| { |
| "schema_version": "ax.native_model.v1", |
| "model_family": "qwen3_5", |
| "tensor_format": "safetensors", |
| "layer_count": 32, |
| "hidden_size": 4096, |
| "intermediate_size": 12288, |
| "attention_head_count": 16, |
| "attention_head_dim": 256, |
| "kv_head_count": 4, |
| "vocab_size": 248320, |
| "tie_word_embeddings": false, |
| "rope_theta": 10000000, |
| "no_rope_layer_interval": 0, |
| "intermediate_size_mlp": 0, |
| "attn_output_gate": true, |
| "partial_rotary_factor": 0.25, |
| "rms_norm_eps": 0.000001, |
| "moe_norm_topk_prob": true, |
| "hidden_size_per_layer_input": 0, |
| "linear_attention": { |
| "full_attention_interval": 4, |
| "num_value_heads": 32, |
| "num_key_heads": 16, |
| "key_head_dim": 128, |
| "value_head_dim": 128, |
| "conv_kernel_dim": 4 |
| }, |
| "think_start_token_id": 248068, |
| "think_end_token_id": 248069, |
| "tensors": [ |
| { |
| "name": "language_model.model.norm.weight", |
| "role": "final_norm", |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 215063840, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.lm_head.weight", |
| "role": "lm_head", |
| "dtype": "bf16", |
| "shape": [ |
| 248320, |
| 4096 |
| ], |
| "file": "model-00002-of-00002.safetensors", |
| "offset_bytes": 142, |
| "length_bytes": 2034237440 |
| }, |
| { |
| "name": "language_model.model.embed_tokens.weight", |
| "role": "token_embedding", |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 8 |
| }, |
| "shape": [ |
| 248320, |
| 1024 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3503499360, |
| "length_bytes": 1017118720 |
| }, |
| { |
| "name": "language_model.model.layers.0.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4843314592, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.0.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1270677216, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4647537696, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4622306336, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.0.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3024420064, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 0, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2144501472, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4807982496, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3422324064, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4674276384, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1994587552, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4791205280, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4854943904, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 0, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4702723104, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.0.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 0, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4526533216, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.1.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2177077088, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.1.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2617584160, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 909339040, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3257533152, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.1.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2319438368, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 1, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4553861728, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4888510624, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3440674208, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3502884640, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2805329056, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2934123744, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2959289568, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 1, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134143840, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.1.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 1, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1645341088, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.2.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4520618080, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.2.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4522723424, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3232359136, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2907385056, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.2.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 647643808, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 2, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4897783968, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3454838304, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4571188192, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3502810912, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2603289120, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1784122208, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3388769632, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 2, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3329966944, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.2.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 2, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1081993888, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3496978208, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3205615840, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.3.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134135520, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3480069920, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.3.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2178150240, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3081575648, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 3, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4854943392, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.3.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2612275744, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3056401632, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3397158240, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.3.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 3, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1705659168, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.4.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3503474464, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.4.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2983423200, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2189193120, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3333702752, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.4.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4863336608, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 4, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134143712, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3303162592, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3142537056, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1730833184, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 941853088, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3286385376, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3222393568, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 4, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2307838240, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.4.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 4, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3123584224, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.5.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2967678176, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.5.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2873298144, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4675914784, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 825227296, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.5.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2756103328, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 5, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3480069792, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3024346336, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1371398176, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 450707936, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1780902752, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3188306144, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134144352, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 5, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2373985312, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.5.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 5, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2987625696, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.6.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2716183456, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.6.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2690485024, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3303228128, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2663746080, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.6.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4816575904, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 6, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4761312544, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1802013536, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2143379104, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2791754912, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2805394592, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2833902752, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3488589600, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 6, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2043002272, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.6.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 6, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2045701792, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1382687200, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2541397536, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.7.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2463270176, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2603878944, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.7.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2461680928, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2640153120, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 7, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2594891808, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.7.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3283763936, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1952251296, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3362555232, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.7.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 7, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2399216928, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.8.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3360973920, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.8.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2344604192, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 587866656, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 613032480, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.8.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1804770144, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 8, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2278461344, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4933968160, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2868513952, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2115980960, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2317275680, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1996750240, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2898996448, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 8, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2752433056, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.8.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 8, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2107592352, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.9.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2177085280, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.9.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4854935200, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1925512608, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 535896608, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.9.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4934033696, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 9, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1416119264, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2373985568, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2178166624, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4702653472, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2273669024, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 853538848, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2179755936, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 9, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3188174816, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.9.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 9, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2594892320, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.10.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2054090400, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.10.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2042994080, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2727267232, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1857731488, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.10.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2080845472, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 10, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2691017504, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2174914400, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1830992736, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2034531744, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2117623456, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2158137184, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 941918624, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 10, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3503482720, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.10.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 10, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2147651424, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 881334304, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 799536672, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.11.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1923931040, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1978006944, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.11.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2578098208, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2017746336, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 11, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1925512096, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.11.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1994653088, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 385368544, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2235330464, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.11.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 11, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1886108576, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.12.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1136520032, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.12.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2143444704, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1755736928, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1830992800, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.12.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2490017312, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 12, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3328393952, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2457937184, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1732668192, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134069984, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1804704608, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2278461472, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4562799584, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 12, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2490017056, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.12.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 12, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2781269152, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.13.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1705642528, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.13.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1697245600, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2996022496, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1672079776, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.13.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1599203744, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 13, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 724927264, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4704296224, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3442779616, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1730898720, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1886043040, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1735813984, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4808048032, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 13, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1705658912, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.13.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 13, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1091955360, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.14.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1519421024, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.14.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1517831776, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1490568288, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4571712544, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.14.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1435518048, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 14, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1100343968, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2263642016, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1581295456, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1582876576, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 561062432, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1554556512, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2950900960, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 14, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1579722336, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.14.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 14, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1571333728, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1414022112, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1517306976, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.15.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1782541152, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4553861856, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.15.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1409950176, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1417167968, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 15, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1410482656, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.15.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1411793888, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2348807200, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1384784352, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.15.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 15, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3454903968, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.16.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 934504864, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.16.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1370341408, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1309909024, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4762893728, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.16.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1270685408, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 16, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1379786848, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1381048800, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3503482656, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1380983264, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1801948000, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3423896992, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1915542432, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 16, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4562258656, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.16.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 16, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1371398240, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.17.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1270144736, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.17.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3024411872, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1218223840, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1244970720, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.17.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1191485152, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 17, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1705642400, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1308262304, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1307451168, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2614635040, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1307541408, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2578106400, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3446449696, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 17, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2688911904, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.17.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 17, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1299062560, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.18.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1184517536, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.18.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2850679968, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2054106784, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1031580064, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.18.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1335074848, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 18, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 693994144, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1977417120, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1185050272, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1911274400, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2727201696, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3205616352, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4925047072, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 18, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1185050016, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.18.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 18, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 523231776, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1157778848, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2348806688, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.19.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1137576800, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1360371744, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.19.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1243389664, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1163021728, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 19, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1137584992, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.19.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2752433312, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2432771360, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1111354208, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.19.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 19, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4899881248, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.20.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1029999008, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.20.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4888502432, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 977045920, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1004833184, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.20.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 950307232, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 20, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1058843040, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4647472160, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1140215136, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1296375584, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1060956704, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1062595104, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1697253792, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 20, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3134144096, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.20.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 20, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1100344096, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.21.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 880277536, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.21.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1370333216, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4736146720, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1465402464, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.21.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 799537184, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 21, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 884025632, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1307467616, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3188174752, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4851199648, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4553796192, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2851212448, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1519429216, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 21, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1380450528, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.21.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 21, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 788002336, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.22.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4892770464, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.22.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 784700832, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 759535008, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2515183136, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.22.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 498000416, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 22, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 786429344, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 786363808, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1296375520, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1988099488, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2143379168, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1140870560, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2569709600, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 22, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 883431456, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.22.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 22, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1586096544, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 729646496, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4523911264, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.23.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3360441440, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2295246880, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.23.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2983415008, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4535446112, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 23, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 726500256, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.23.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 727549344, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 698188576, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3098418400, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.23.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 23, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 732792224, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.24.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 638198304, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.24.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2295238688, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 562700832, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 884165024, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.24.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2463278368, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 24, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1309908896, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3098352864, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1307541344, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 330096736, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 676561568, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 677216928, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2265280416, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 24, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3360982112, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.24.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 24, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 639255200, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.25.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 497983968, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.25.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 497967584, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3159846816, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 471753184, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.25.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2873306336, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 25, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 639255072, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 784709024, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 497992160, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 523166240, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 532152864, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4959199520, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2424382752, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 25, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4843855264, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.25.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 25, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2717764512, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.26.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3503491168, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.26.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3021188320, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 354279520, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2542970912, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.26.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2807032992, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 26, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 385352032, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2045632160, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1111354144, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3333637216, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 676627104, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 450773472, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 442319328, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 26, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 381673568, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.26.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 26, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1663691168, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 326942816, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 325500512, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.27.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 325492320, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2974502112, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.27.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1913961376, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 330162272, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 27, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1586096032, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.27.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 672883360, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 297180768, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2691017632, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.27.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 27, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4596878368, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.28.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4893958304, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.28.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 274922784, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 246611232, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2118213280, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.28.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 218299680, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 28, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3454903840, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 293961312, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 283319840, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 283336288, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 283410016, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1624369568, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 283475552, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 28, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 283319584, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.28.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 28, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 274930976, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.29.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3188158368, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.29.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 206068704, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 177757152, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1529390688, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.29.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 152591328, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 29, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1080945184, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 216661280, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 214990048, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 214998304, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 883956000, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3143069600, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2795367584, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 29, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 214989792, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.29.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 29, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 206601184, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.30.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 138935072, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.30.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4562258912, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 112196384, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2374051104, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.30.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 83884832, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.A_log", |
| "role": "linear_attention_a_log", |
| "layer_index": 30, |
| "dtype": "f32", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 139991840, |
| "length_bytes": 128 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.conv1d.weight", |
| "role": "linear_attention_conv1d", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 8192, |
| 4, |
| 1 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 672817824, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.dt_bias", |
| "role": "linear_attention_dt_bias", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 32 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 139991968, |
| "length_bytes": 64 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_a.weight", |
| "role": "linear_attention_in_proj_a", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 3500648224, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_b.weight", |
| "role": "linear_attention_in_proj_b", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 32, |
| "bits": 4 |
| }, |
| "shape": [ |
| 32, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2873232608, |
| "length_bytes": 65536 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_qkv.weight", |
| "role": "linear_attention_in_proj_qkv", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2214358944, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.in_proj_z.weight", |
| "role": "linear_attention_in_proj_z", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 140008416, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.norm.weight", |
| "role": "linear_attention_norm", |
| "layer_index": 30, |
| "dtype": "bf16", |
| "shape": [ |
| 128 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2305740832, |
| "length_bytes": 256 |
| }, |
| { |
| "name": "language_model.model.layers.30.linear_attn.out_proj.weight", |
| "role": "linear_attention_out_proj", |
| "layer_index": 30, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 871888928, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.k_proj.weight", |
| "role": "attention_k", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2967686368, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.k_norm.weight", |
| "role": "attention_k_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 80344864, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.31.input_layernorm.weight", |
| "role": "attention_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 533266976, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.o_proj.weight", |
| "role": "attention_o", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1655302560, |
| "length_bytes": 8388608 |
| }, |
| { |
| "name": "language_model.model.layers.31.post_attention_layernorm.weight", |
| "role": "attention_post_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 4096 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 80336672, |
| "length_bytes": 8192 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.q_proj.weight", |
| "role": "attention_q", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 8192, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 2618640928, |
| "length_bytes": 16777216 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.q_norm.weight", |
| "role": "attention_q_norm", |
| "layer_index": 31, |
| "dtype": "bf16", |
| "shape": [ |
| 256 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 80345376, |
| "length_bytes": 512 |
| }, |
| { |
| "name": "language_model.model.layers.31.self_attn.v_proj.weight", |
| "role": "attention_v", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 1024, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 4520626272, |
| "length_bytes": 2097152 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.down_proj.weight", |
| "role": "ffn_down", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 4096, |
| 1536 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 26859296, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.gate_proj.weight", |
| "role": "ffn_gate", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 55170848, |
| "length_bytes": 25165824 |
| }, |
| { |
| "name": "language_model.model.layers.31.mlp.up_proj.weight", |
| "role": "ffn_up", |
| "layer_index": 31, |
| "dtype": "u32", |
| "source_quantized": true, |
| "quantization": { |
| "mode": "affine", |
| "group_size": 64, |
| "bits": 4 |
| }, |
| "shape": [ |
| 12288, |
| 512 |
| ], |
| "file": "model-00001-of-00002.safetensors", |
| "offset_bytes": 1693472, |
| "length_bytes": 25165824 |
| } |
| ] |
| } |
