Self-quantized qwen2.5-1.5b-gptq 4-bit (group_size=128, seed=42)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,103 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "pad_token_id": 151643,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "failsafe": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "./gptqmodel_offload/vkhdvqgl-wrenvzpi/",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:5.8.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "vram_strategy": "exclusive",
+      "wait_for_submodule_finalizers": false
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.4.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.4.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48541182c22e9f9d688678b88e98c39633f4020799370e9e614d3953b5af035
+size 1149862904

quant_log.csv

@@ -0,0 +1,197 @@
+layer,module,loss,samples,damp,time
+0,self_attn.q_proj,0.0001942568,0.05000,3.519
+0,self_attn.v_proj,0.0000033538,0.05000,3.620
+0,self_attn.k_proj,0.0000293859,0.05000,3.614
+0,self_attn.o_proj,0.0000074849,0.05000,0.828
+0,mlp.gate_proj,0.0001148074,0.05000,1.608
+0,mlp.up_proj,0.0000794248,0.05000,1.635
+0,mlp.down_proj,0.0000133268,0.05000,4.222
+1,self_attn.v_proj,0.0000021513,0.05000,3.167
+1,self_attn.q_proj,0.0000325077,0.05000,3.201
+1,self_attn.k_proj,0.0000094605,0.05000,3.233
+1,self_attn.o_proj,0.0000030150,0.05000,0.610
+1,mlp.gate_proj,0.0025643470,0.05000,1.753
+1,mlp.up_proj,0.0020986227,0.05000,1.772
+1,mlp.down_proj,0.0009006713,0.05000,3.821
+2,self_attn.v_proj,0.0000084684,0.05000,3.665
+2,self_attn.q_proj,0.0000954910,0.05000,3.666
+2,self_attn.k_proj,0.0000202702,0.05000,3.718
+2,self_attn.o_proj,0.0000040788,0.05000,0.575
+2,mlp.gate_proj,0.0021059955,0.05000,1.774
+2,mlp.up_proj,0.0013822045,0.05000,1.819
+2,mlp.down_proj,0.0008856719,0.05000,3.623
+3,self_attn.q_proj,0.0000963069,0.05000,3.376
+3,self_attn.k_proj,0.0000205040,0.05000,3.453
+3,self_attn.v_proj,0.0000115286,0.05000,3.487
+3,self_attn.o_proj,0.0000066182,0.05000,0.693
+3,mlp.gate_proj,0.0016237398,0.05000,1.920
+3,mlp.up_proj,0.0008927437,0.05000,1.933
+3,mlp.down_proj,0.0000440674,0.05000,3.765
+4,self_attn.k_proj,0.0000238032,0.05000,3.398
+4,self_attn.q_proj,0.0001242649,0.05000,3.501
+4,self_attn.v_proj,0.0000161745,0.05000,3.530
+4,self_attn.o_proj,0.0000089664,0.05000,0.618
+4,mlp.gate_proj,0.0010840711,0.05000,1.829
+4,mlp.up_proj,0.0006642618,0.05000,1.856
+4,mlp.down_proj,0.0000501768,0.05000,3.848
+5,self_attn.v_proj,0.0000227581,0.05000,3.174
+5,self_attn.q_proj,0.0001299048,0.05000,3.270
+5,self_attn.k_proj,0.0000275618,0.05000,3.280
+5,self_attn.o_proj,0.0000138769,0.05000,0.670
+5,mlp.up_proj,0.0013722564,0.05000,1.851
+5,mlp.gate_proj,0.0020988442,0.05000,1.863
+5,mlp.down_proj,0.0000409705,0.05000,4.404
+6,self_attn.v_proj,0.0000188698,0.05000,3.315
+6,self_attn.q_proj,0.0001721964,0.05000,3.351
+6,self_attn.k_proj,0.0000359508,0.05000,3.383
+6,self_attn.o_proj,0.0000058918,0.05000,0.593
+6,mlp.gate_proj,0.0005833350,0.05000,1.583
+6,mlp.up_proj,0.0004894385,0.05000,1.611
+6,mlp.down_proj,0.0000467236,0.05000,3.904
+7,self_attn.v_proj,0.0000124919,0.05000,3.632
+7,self_attn.q_proj,0.0000809716,0.05000,3.632
+7,self_attn.k_proj,0.0000157959,0.05000,3.803
+7,self_attn.o_proj,0.0000089062,0.05000,0.716
+7,mlp.up_proj,0.0004703383,0.05000,1.955
+7,mlp.gate_proj,0.0004911946,0.05000,1.972
+7,mlp.down_proj,0.0000479692,0.05000,4.039
+8,self_attn.q_proj,0.0001606633,0.05000,3.426
+8,self_attn.k_proj,0.0000285278,0.05000,3.517
+8,self_attn.v_proj,0.0000176461,0.05000,3.571
+8,self_attn.o_proj,0.0000114544,0.05000,0.660
+8,mlp.gate_proj,0.0004976164,0.05000,2.604
+8,mlp.up_proj,0.0004567402,0.05000,2.623
+8,mlp.down_proj,0.0000424818,0.05000,4.143
+9,self_attn.v_proj,0.0000145518,0.05000,3.436
+9,self_attn.q_proj,0.0001237559,0.05000,3.441
+9,self_attn.k_proj,0.0000253094,0.05000,3.456
+9,self_attn.o_proj,0.0000114307,0.05000,0.677
+9,mlp.gate_proj,0.0004502635,0.05000,1.684
+9,mlp.up_proj,0.0004355869,0.05000,1.763
+9,mlp.down_proj,0.0000410696,0.05000,3.787
+10,self_attn.v_proj,0.0000207346,0.05000,4.259
+10,self_attn.k_proj,0.0000242412,0.05000,4.348
+10,self_attn.q_proj,0.0001243170,0.05000,4.352
+10,self_attn.o_proj,0.0000184077,0.05000,0.714
+10,mlp.gate_proj,0.0004360236,0.05000,2.063
+10,mlp.up_proj,0.0004101034,0.05000,2.141
+10,mlp.down_proj,0.0000363149,0.05000,3.978
+11,self_attn.q_proj,0.0001229121,0.05000,3.551
+11,self_attn.v_proj,0.0000171363,0.05000,3.682
+11,self_attn.k_proj,0.0000240140,0.05000,3.698
+11,self_attn.o_proj,0.0000168354,0.05000,0.696
+11,mlp.gate_proj,0.0004277187,0.05000,1.717
+11,mlp.up_proj,0.0003753171,0.05000,1.802
+11,mlp.down_proj,0.0000307441,0.05000,3.849
+12,self_attn.v_proj,0.0000160176,0.05000,3.036
+12,self_attn.k_proj,0.0000313926,0.05000,3.061
+12,self_attn.q_proj,0.0001444434,0.05000,3.129
+12,self_attn.o_proj,0.0000115865,0.05000,0.586
+12,mlp.up_proj,0.0003572416,0.05000,1.614
+12,mlp.gate_proj,0.0003940164,0.05000,1.674
+12,mlp.down_proj,0.0000321534,0.05000,3.836
+13,self_attn.k_proj,0.0000188857,0.05000,3.238
+13,self_attn.q_proj,0.0000945165,0.05000,3.262
+13,self_attn.v_proj,0.0000123991,0.05000,3.287
+13,self_attn.o_proj,0.0000148398,0.05000,0.745
+13,mlp.up_proj,0.0003421897,0.05000,1.722
+13,mlp.gate_proj,0.0003560762,0.05000,1.794
+13,mlp.down_proj,0.0000275840,0.05000,4.004
+14,self_attn.v_proj,0.0000249311,0.05000,3.388
+14,self_attn.q_proj,0.0002223736,0.05000,3.478
+14,self_attn.k_proj,0.0000331152,0.05000,3.506
+14,self_attn.o_proj,0.0000111160,0.05000,0.599
+14,mlp.gate_proj,0.0003798449,0.05000,1.741
+14,mlp.up_proj,0.0003853373,0.05000,1.800
+14,mlp.down_proj,0.0000325325,0.05000,4.420
+15,self_attn.k_proj,0.0000236891,0.05000,3.697
+15,self_attn.v_proj,0.0000209465,0.05000,3.741
+15,self_attn.q_proj,0.0002121266,0.05000,3.752
+15,self_attn.o_proj,0.0000193666,0.05000,0.624
+15,mlp.gate_proj,0.0003494259,0.05000,1.794
+15,mlp.up_proj,0.0003297445,0.05000,1.827
+15,mlp.down_proj,0.0000274264,0.05000,4.004
+16,self_attn.k_proj,0.0000308558,0.05000,3.403
+16,self_attn.v_proj,0.0000237038,0.05000,3.440
+16,self_attn.q_proj,0.0001626623,0.05000,3.475
+16,self_attn.o_proj,0.0000150444,0.05000,0.622
+16,mlp.gate_proj,0.0003889125,0.05000,1.860
+16,mlp.up_proj,0.0003813872,0.05000,1.896
+16,mlp.down_proj,0.0000318470,0.05000,4.405
+17,self_attn.q_proj,0.0001462495,0.05000,3.408
+17,self_attn.v_proj,0.0000311132,0.05000,3.492
+17,self_attn.k_proj,0.0000180292,0.05000,3.489
+17,self_attn.o_proj,0.0000120925,0.05000,0.696
+17,mlp.gate_proj,0.0003979664,0.05000,2.337
+17,mlp.up_proj,0.0003952374,0.05000,2.380
+17,mlp.down_proj,0.0000415275,0.05000,3.937
+18,self_attn.q_proj,0.0001291138,0.05000,3.377
+18,self_attn.k_proj,0.0000215902,0.05000,3.553
+18,self_attn.v_proj,0.0000218364,0.05000,3.576
+18,self_attn.o_proj,0.0000215382,0.05000,0.656
+18,mlp.gate_proj,0.0004375501,0.05000,1.857
+18,mlp.up_proj,0.0004303935,0.05000,1.867
+18,mlp.down_proj,0.0000551119,0.05000,4.582
+19,self_attn.v_proj,0.0000468919,0.05000,3.290
+19,self_attn.q_proj,0.0001769957,0.05000,3.319
+19,self_attn.k_proj,0.0000234716,0.05000,3.392
+19,self_attn.o_proj,0.0000198349,0.05000,0.578
+19,mlp.gate_proj,0.0004960430,0.05000,2.169
+19,mlp.up_proj,0.0005314743,0.05000,2.238
+19,mlp.down_proj,0.0001041916,0.05000,4.563
+20,self_attn.k_proj,0.0000279328,0.05000,3.557
+20,self_attn.q_proj,0.0002208864,0.05000,3.645
+20,self_attn.v_proj,0.0000660211,0.05000,3.669
+20,self_attn.o_proj,0.0000338332,0.05000,0.588
+20,mlp.up_proj,0.0005629465,0.05000,1.994
+20,mlp.gate_proj,0.0005495163,0.05000,1.995
+20,mlp.down_proj,0.0000975834,0.05000,4.368
+21,self_attn.v_proj,0.0000588163,0.05000,3.694
+21,self_attn.q_proj,0.0002200743,0.05000,3.721
+21,self_attn.k_proj,0.0000272952,0.05000,3.829
+21,self_attn.o_proj,0.0000264764,0.05000,0.686
+21,mlp.gate_proj,0.0007878635,0.05000,2.148
+21,mlp.up_proj,0.0007888174,0.05000,2.247
+21,mlp.down_proj,0.0001698532,0.05000,3.959
+22,self_attn.q_proj,0.0002093418,0.05000,3.808
+22,self_attn.k_proj,0.0000318649,0.05000,3.851
+22,self_attn.v_proj,0.0000711933,0.05000,3.869
+22,self_attn.o_proj,0.0000268098,0.05000,0.768
+22,mlp.gate_proj,0.0009505198,0.05000,1.796
+22,mlp.up_proj,0.0009412312,0.05000,1.825
+22,mlp.down_proj,0.0002310233,0.05000,4.523
+23,self_attn.v_proj,0.0001066141,0.05000,4.407
+23,self_attn.q_proj,0.0003122848,0.05000,4.404
+23,self_attn.k_proj,0.0000369567,0.05000,4.458
+23,self_attn.o_proj,0.0000279494,0.05000,0.759
+23,mlp.up_proj,0.0010348070,0.05000,2.857
+23,mlp.gate_proj,0.0009924937,0.05000,2.870
+23,mlp.down_proj,0.0002910868,0.05000,5.011
+24,self_attn.v_proj,0.0001675055,0.05000,3.922
+24,self_attn.k_proj,0.0000384572,0.05000,4.003
+24,self_attn.q_proj,0.0002870688,0.05000,4.012
+24,self_attn.o_proj,0.0000425614,0.05000,0.735
+24,mlp.up_proj,0.0011134829,0.05000,2.437
+24,mlp.gate_proj,0.0010364587,0.05000,2.444
+24,mlp.down_proj,0.0003481629,0.05000,4.686
+25,self_attn.q_proj,0.0003447584,0.05000,4.070
+25,self_attn.k_proj,0.0000364399,0.05000,4.078
+25,self_attn.v_proj,0.0001667999,0.05000,4.065
+25,self_attn.o_proj,0.0000568331,0.05000,0.845
+25,mlp.gate_proj,0.0010463374,0.05000,1.863
+25,mlp.up_proj,0.0012092180,0.05000,1.966
+25,mlp.down_proj,0.0004537338,0.05000,4.291
+26,self_attn.v_proj,0.0002388664,0.05000,3.937
+26,self_attn.q_proj,0.0003158843,0.05000,4.007
+26,self_attn.k_proj,0.0000400289,0.05000,4.045
+26,self_attn.o_proj,0.0000527040,0.05000,0.690
+26,mlp.up_proj,0.0012510429,0.05000,2.057
+26,mlp.gate_proj,0.0010409112,0.05000,2.327
+26,mlp.down_proj,0.0020226345,0.05000,4.384
+27,self_attn.v_proj,0.0002370400,0.05000,4.069
+27,self_attn.q_proj,0.0003039186,0.05000,4.089
+27,self_attn.k_proj,0.0000327215,0.05000,4.103
+27,self_attn.o_proj,0.0001643453,0.05000,0.791
+27,mlp.gate_proj,0.0015339861,0.05000,2.210
+27,mlp.up_proj,0.0015848841,0.05000,2.267
+27,mlp.down_proj,0.0012275284,0.05000,4.896

quantize_config.json

@@ -0,0 +1,7 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "sym": true,
+  "quant_method": "gptq"
+}

quantize_manifest.json

@@ -0,0 +1,12 @@
+{
+  "model_name": "qwen2.5-1.5b",
+  "hf_id": "Qwen/Qwen2.5-1.5B-Instruct",
+  "method": "gptq",
+  "bits": 4,
+  "group_size": 128,
+  "calibration_dataset": "allenai/c4",
+  "calibration_samples": 128,
+  "seed": 42,
+  "elapsed_s": 554.7059471607208,
+  "tool": "gptqmodel"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,30 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "<|im_start|>": "<|im_start|>",
+    "<|im_end|>": "<|im_end|>",
+    "<|object_ref_start|>": "<|object_ref_start|>",
+    "<|object_ref_end|>": "<|object_ref_end|>",
+    "<|box_start|>": "<|box_start|>",
+    "<|box_end|>": "<|box_end|>",
+    "<|quad_start|>": "<|quad_start|>",
+    "<|quad_end|>": "<|quad_end|>",
+    "<|vision_start|>": "<|vision_start|>",
+    "<|vision_end|>": "<|vision_end|>",
+    "<|vision_pad|>": "<|vision_pad|>",
+    "<|image_pad|>": "<|image_pad|>",
+    "<|video_pad|>": "<|video_pad|>"
+  },
+  "is_local": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2TokenizerFast",
+  "unk_token": null,
+  "_commit_hash": null
+}