diff --git "a/model-manifest.json" "b/model-manifest.json" new file mode 100644--- /dev/null +++ "b/model-manifest.json" @@ -0,0 +1,6946 @@ +{ + "schema_version": "ax.native_model.v1", + "model_family": "qwen3_5", + "tensor_format": "safetensors", + "layer_count": 32, + "hidden_size": 4096, + "intermediate_size": 12288, + "attention_head_count": 16, + "attention_head_dim": 256, + "kv_head_count": 4, + "vocab_size": 248320, + "tie_word_embeddings": false, + "rope_theta": 10000000, + "no_rope_layer_interval": 0, + "intermediate_size_mlp": 0, + "attn_output_gate": true, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 0.000001, + "moe_norm_topk_prob": true, + "hidden_size_per_layer_input": 0, + "linear_attention": { + "full_attention_interval": 4, + "num_value_heads": 32, + "num_key_heads": 16, + "key_head_dim": 128, + "value_head_dim": 128, + "conv_kernel_dim": 4 + }, + "think_start_token_id": 248068, + "think_end_token_id": 248069, + "tensors": [ + { + "name": "language_model.model.norm.weight", + "role": "final_norm", + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 215063840, + "length_bytes": 8192 + }, + { + "name": "language_model.lm_head.weight", + "role": "lm_head", + "dtype": "bf16", + "shape": [ + 248320, + 4096 + ], + "file": "model-00002-of-00002.safetensors", + "offset_bytes": 142, + "length_bytes": 2034237440 + }, + { + "name": "language_model.model.embed_tokens.weight", + "role": "token_embedding", + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 8 + }, + "shape": [ + 248320, + 1024 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3503499360, + "length_bytes": 1017118720 + }, + { + "name": "language_model.model.layers.0.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 0, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4843314592, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.0.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 0, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1270677216, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.0.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4647537696, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.0.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4622306336, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.0.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3024420064, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.0.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 0, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2144501472, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.0.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 0, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4807982496, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.0.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 0, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3422324064, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.0.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4674276384, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.0.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1994587552, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.0.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4791205280, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.0.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4854943904, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.0.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 0, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4702723104, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.0.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 0, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4526533216, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.1.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 1, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2177077088, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.1.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 1, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2617584160, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.1.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 909339040, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.1.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3257533152, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.1.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2319438368, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.1.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 1, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4553861728, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.1.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 1, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4888510624, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.1.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 1, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3440674208, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.1.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3502884640, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.1.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2805329056, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.1.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2934123744, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.1.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2959289568, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.1.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 1, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134143840, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.1.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 1, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1645341088, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.2.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 2, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4520618080, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.2.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 2, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4522723424, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.2.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3232359136, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.2.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2907385056, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.2.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 647643808, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.2.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 2, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4897783968, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.2.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 2, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3454838304, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.2.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 2, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4571188192, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.2.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3502810912, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.2.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2603289120, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.2.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1784122208, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.2.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3388769632, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.2.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 2, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3329966944, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.2.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 2, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1081993888, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.3.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3496978208, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.3.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 3, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3205615840, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.3.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 3, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134135520, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.3.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3480069920, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.3.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 3, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2178150240, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.3.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3081575648, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.3.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 3, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4854943392, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.3.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2612275744, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.3.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3056401632, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.3.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3397158240, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.3.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 3, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1705659168, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.4.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 4, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3503474464, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.4.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 4, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2983423200, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.4.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2189193120, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.4.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3333702752, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.4.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4863336608, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.4.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 4, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134143712, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.4.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 4, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3303162592, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.4.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 4, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3142537056, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.4.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1730833184, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.4.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 941853088, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.4.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3286385376, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.4.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3222393568, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.4.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 4, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2307838240, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.4.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 4, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3123584224, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.5.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 5, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2967678176, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.5.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 5, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2873298144, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.5.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4675914784, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.5.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 825227296, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.5.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2756103328, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.5.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 5, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3480069792, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.5.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 5, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3024346336, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.5.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 5, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1371398176, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.5.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 450707936, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.5.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1780902752, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.5.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3188306144, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.5.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134144352, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.5.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 5, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2373985312, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.5.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 5, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2987625696, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.6.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 6, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2716183456, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.6.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 6, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2690485024, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.6.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3303228128, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.6.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2663746080, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.6.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4816575904, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.6.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 6, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4761312544, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.6.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 6, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1802013536, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.6.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 6, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2143379104, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.6.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2791754912, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.6.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2805394592, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.6.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2833902752, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.6.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3488589600, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.6.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 6, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2043002272, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.6.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 6, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2045701792, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.7.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1382687200, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.7.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 7, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2541397536, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.7.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 7, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2463270176, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.7.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2603878944, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.7.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 7, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2461680928, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.7.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2640153120, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.7.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 7, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2594891808, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.7.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3283763936, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.7.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1952251296, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.7.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3362555232, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.7.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 7, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2399216928, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.8.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 8, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3360973920, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.8.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 8, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2344604192, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.8.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 587866656, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.8.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 613032480, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.8.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1804770144, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.8.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 8, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2278461344, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.8.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 8, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4933968160, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.8.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 8, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2868513952, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.8.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2115980960, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.8.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2317275680, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.8.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1996750240, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.8.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2898996448, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.8.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 8, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2752433056, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.8.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 8, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2107592352, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.9.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 9, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2177085280, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.9.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 9, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4854935200, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.9.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1925512608, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.9.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 535896608, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.9.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4934033696, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.9.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 9, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1416119264, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.9.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 9, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2373985568, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.9.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 9, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2178166624, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.9.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4702653472, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.9.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2273669024, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.9.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 853538848, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.9.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2179755936, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.9.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 9, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3188174816, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.9.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 9, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2594892320, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.10.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 10, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2054090400, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.10.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 10, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2042994080, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.10.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2727267232, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.10.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1857731488, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.10.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2080845472, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.10.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 10, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2691017504, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.10.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 10, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2174914400, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.10.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 10, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1830992736, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.10.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2034531744, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.10.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2117623456, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.10.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2158137184, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.10.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 941918624, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.10.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 10, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3503482720, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.10.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 10, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2147651424, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.11.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 881334304, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.11.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 11, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 799536672, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.11.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 11, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1923931040, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.11.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1978006944, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.11.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 11, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2578098208, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.11.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2017746336, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.11.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 11, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1925512096, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.11.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1994653088, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.11.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 385368544, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.11.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2235330464, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.11.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 11, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1886108576, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.12.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 12, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1136520032, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.12.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 12, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2143444704, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.12.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1755736928, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.12.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1830992800, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.12.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2490017312, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.12.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 12, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3328393952, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.12.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 12, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2457937184, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.12.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 12, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1732668192, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.12.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134069984, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.12.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1804704608, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.12.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2278461472, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.12.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4562799584, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.12.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 12, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2490017056, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.12.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 12, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2781269152, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.13.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 13, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1705642528, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.13.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 13, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1697245600, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.13.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2996022496, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.13.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1672079776, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.13.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1599203744, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.13.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 13, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 724927264, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.13.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 13, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4704296224, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.13.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 13, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3442779616, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.13.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1730898720, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.13.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1886043040, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.13.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1735813984, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.13.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4808048032, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.13.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 13, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1705658912, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.13.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 13, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1091955360, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.14.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 14, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1519421024, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.14.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 14, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1517831776, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.14.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1490568288, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.14.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4571712544, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.14.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1435518048, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.14.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 14, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1100343968, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.14.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 14, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2263642016, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.14.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 14, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1581295456, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.14.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1582876576, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.14.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 561062432, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.14.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1554556512, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.14.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2950900960, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.14.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 14, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1579722336, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.14.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 14, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1571333728, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.15.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1414022112, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.15.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 15, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1517306976, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.15.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 15, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1782541152, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.15.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4553861856, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.15.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 15, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1409950176, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.15.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1417167968, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.15.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 15, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1410482656, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.15.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1411793888, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.15.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2348807200, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.15.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1384784352, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.15.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 15, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3454903968, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.16.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 16, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 934504864, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.16.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 16, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1370341408, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.16.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1309909024, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.16.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4762893728, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.16.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1270685408, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.16.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 16, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1379786848, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.16.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 16, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1381048800, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.16.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 16, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3503482656, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.16.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1380983264, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.16.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1801948000, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.16.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3423896992, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.16.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1915542432, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.16.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 16, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4562258656, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.16.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 16, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1371398240, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.17.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 17, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1270144736, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.17.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 17, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3024411872, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.17.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1218223840, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.17.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1244970720, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.17.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1191485152, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.17.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 17, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1705642400, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.17.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 17, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1308262304, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.17.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 17, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1307451168, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.17.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2614635040, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.17.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1307541408, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.17.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2578106400, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.17.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3446449696, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.17.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 17, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2688911904, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.17.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 17, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1299062560, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.18.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 18, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1184517536, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.18.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 18, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2850679968, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.18.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2054106784, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.18.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1031580064, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.18.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1335074848, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.18.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 18, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 693994144, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.18.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 18, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1977417120, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.18.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 18, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1185050272, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.18.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1911274400, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.18.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2727201696, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.18.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3205616352, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.18.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4925047072, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.18.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 18, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1185050016, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.18.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 18, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 523231776, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.19.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1157778848, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.19.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 19, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2348806688, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.19.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 19, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1137576800, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.19.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1360371744, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.19.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 19, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1243389664, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.19.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1163021728, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.19.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 19, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1137584992, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.19.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2752433312, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.19.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2432771360, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.19.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1111354208, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.19.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 19, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4899881248, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.20.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 20, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1029999008, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.20.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 20, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4888502432, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.20.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 977045920, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.20.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1004833184, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.20.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 950307232, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.20.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 20, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1058843040, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.20.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 20, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4647472160, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.20.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 20, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1140215136, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.20.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1296375584, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.20.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1060956704, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.20.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1062595104, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.20.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1697253792, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.20.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 20, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3134144096, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.20.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 20, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1100344096, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.21.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 21, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 880277536, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.21.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 21, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1370333216, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.21.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4736146720, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.21.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1465402464, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.21.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 799537184, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.21.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 21, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 884025632, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.21.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 21, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1307467616, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.21.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 21, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3188174752, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.21.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4851199648, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.21.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4553796192, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.21.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2851212448, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.21.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1519429216, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.21.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 21, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1380450528, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.21.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 21, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 788002336, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.22.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 22, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4892770464, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.22.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 22, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 784700832, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.22.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 759535008, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.22.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2515183136, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.22.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 498000416, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.22.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 22, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 786429344, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.22.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 22, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 786363808, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.22.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 22, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1296375520, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.22.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1988099488, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.22.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2143379168, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.22.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1140870560, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.22.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2569709600, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.22.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 22, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 883431456, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.22.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 22, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1586096544, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.23.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 729646496, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.23.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 23, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4523911264, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.23.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 23, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3360441440, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.23.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2295246880, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.23.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 23, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2983415008, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.23.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4535446112, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.23.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 23, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 726500256, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.23.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 727549344, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.23.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 698188576, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.23.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3098418400, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.23.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 23, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 732792224, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.24.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 24, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 638198304, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.24.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 24, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2295238688, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.24.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 562700832, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.24.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 884165024, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.24.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2463278368, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.24.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 24, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1309908896, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.24.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 24, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3098352864, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.24.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 24, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1307541344, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.24.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 330096736, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.24.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 676561568, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.24.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 677216928, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.24.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2265280416, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.24.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 24, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3360982112, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.24.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 24, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 639255200, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.25.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 25, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 497983968, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.25.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 25, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 497967584, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.25.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3159846816, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.25.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 471753184, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.25.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2873306336, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.25.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 25, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 639255072, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.25.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 25, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 784709024, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.25.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 25, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 497992160, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.25.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 523166240, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.25.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 532152864, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.25.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4959199520, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.25.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2424382752, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.25.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 25, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4843855264, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.25.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 25, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2717764512, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.26.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 26, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3503491168, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.26.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 26, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3021188320, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.26.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 354279520, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.26.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2542970912, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.26.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2807032992, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.26.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 26, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 385352032, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.26.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 26, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2045632160, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.26.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 26, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1111354144, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.26.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3333637216, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.26.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 676627104, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.26.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 450773472, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.26.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 442319328, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.26.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 26, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 381673568, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.26.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 26, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1663691168, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.27.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 326942816, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.27.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 27, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 325500512, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.27.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 27, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 325492320, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.27.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2974502112, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.27.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 27, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1913961376, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.27.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 330162272, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.27.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 27, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1586096032, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.27.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 672883360, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.27.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 297180768, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.27.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2691017632, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.27.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 27, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4596878368, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.28.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 28, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4893958304, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.28.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 28, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 274922784, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.28.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 246611232, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.28.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2118213280, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.28.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 218299680, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.28.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 28, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3454903840, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.28.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 28, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 293961312, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.28.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 28, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 283319840, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.28.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 283336288, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.28.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 283410016, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.28.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1624369568, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.28.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 283475552, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.28.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 28, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 283319584, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.28.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 28, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 274930976, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.29.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 29, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3188158368, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.29.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 29, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 206068704, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.29.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 177757152, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.29.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1529390688, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.29.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 152591328, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.29.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 29, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1080945184, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.29.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 29, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 216661280, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.29.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 29, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 214990048, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.29.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 214998304, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.29.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 883956000, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.29.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3143069600, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.29.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2795367584, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.29.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 29, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 214989792, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.29.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 29, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 206601184, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.30.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 30, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 138935072, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.30.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 30, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4562258912, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.30.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 112196384, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.30.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2374051104, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.30.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 83884832, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.30.linear_attn.A_log", + "role": "linear_attention_a_log", + "layer_index": 30, + "dtype": "f32", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 139991840, + "length_bytes": 128 + }, + { + "name": "language_model.model.layers.30.linear_attn.conv1d.weight", + "role": "linear_attention_conv1d", + "layer_index": 30, + "dtype": "bf16", + "shape": [ + 8192, + 4, + 1 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 672817824, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.30.linear_attn.dt_bias", + "role": "linear_attention_dt_bias", + "layer_index": 30, + "dtype": "bf16", + "shape": [ + 32 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 139991968, + "length_bytes": 64 + }, + { + "name": "language_model.model.layers.30.linear_attn.in_proj_a.weight", + "role": "linear_attention_in_proj_a", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 3500648224, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.30.linear_attn.in_proj_b.weight", + "role": "linear_attention_in_proj_b", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 32, + "bits": 4 + }, + "shape": [ + 32, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2873232608, + "length_bytes": 65536 + }, + { + "name": "language_model.model.layers.30.linear_attn.in_proj_qkv.weight", + "role": "linear_attention_in_proj_qkv", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2214358944, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.30.linear_attn.in_proj_z.weight", + "role": "linear_attention_in_proj_z", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 140008416, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.30.linear_attn.norm.weight", + "role": "linear_attention_norm", + "layer_index": 30, + "dtype": "bf16", + "shape": [ + 128 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2305740832, + "length_bytes": 256 + }, + { + "name": "language_model.model.layers.30.linear_attn.out_proj.weight", + "role": "linear_attention_out_proj", + "layer_index": 30, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 871888928, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.31.self_attn.k_proj.weight", + "role": "attention_k", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2967686368, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.31.self_attn.k_norm.weight", + "role": "attention_k_norm", + "layer_index": 31, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 80344864, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.31.input_layernorm.weight", + "role": "attention_norm", + "layer_index": 31, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 533266976, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.31.self_attn.o_proj.weight", + "role": "attention_o", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1655302560, + "length_bytes": 8388608 + }, + { + "name": "language_model.model.layers.31.post_attention_layernorm.weight", + "role": "attention_post_norm", + "layer_index": 31, + "dtype": "bf16", + "shape": [ + 4096 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 80336672, + "length_bytes": 8192 + }, + { + "name": "language_model.model.layers.31.self_attn.q_proj.weight", + "role": "attention_q", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 8192, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 2618640928, + "length_bytes": 16777216 + }, + { + "name": "language_model.model.layers.31.self_attn.q_norm.weight", + "role": "attention_q_norm", + "layer_index": 31, + "dtype": "bf16", + "shape": [ + 256 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 80345376, + "length_bytes": 512 + }, + { + "name": "language_model.model.layers.31.self_attn.v_proj.weight", + "role": "attention_v", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 1024, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 4520626272, + "length_bytes": 2097152 + }, + { + "name": "language_model.model.layers.31.mlp.down_proj.weight", + "role": "ffn_down", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 4096, + 1536 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 26859296, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.31.mlp.gate_proj.weight", + "role": "ffn_gate", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 55170848, + "length_bytes": 25165824 + }, + { + "name": "language_model.model.layers.31.mlp.up_proj.weight", + "role": "ffn_up", + "layer_index": 31, + "dtype": "u32", + "source_quantized": true, + "quantization": { + "mode": "affine", + "group_size": 64, + "bits": 4 + }, + "shape": [ + 12288, + 512 + ], + "file": "model-00001-of-00002.safetensors", + "offset_bytes": 1693472, + "length_bytes": 25165824 + } + ] +} \ No newline at end of file