Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,276 @@
+---
+library_name: transformers
+base_model:
+- zai-org/GLM-5
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [zai-org/GLM-5](https://huggingface.co/zai-org/GLM-5).
+
+| File path | Size |
+|------|------|
+| model.safetensors | 9.0MB |
+
+
+### Example usage:
+
+- vLLM
+
+```bash
+# Multi-token prediction is supported
+model_id=tiny-random/glm-moe-dsa
+vllm serve $model_id \
+     --tensor-parallel-size 2 \
+     --speculative-config.method mtp \
+     --speculative-config.num_speculative_tokens 1 \
+     --tool-call-parser glm47 \
+     --reasoning-parser glm45 \
+     --enable-auto-tool-choice
+```
+
+- SGLang
+
+```bash
+# Multi-token prediction is supported
+model_id=tiny-random/glm-moe-dsa
+python3 -m sglang.launch_server   --model-path $model_id   --tp-size 2 \
+  --tool-call-parser glm47  \
+  --reasoning-parser glm45 \
+  --speculative-algorithm EAGLE \
+  --speculative-num-steps 3 \
+  --speculative-eagle-topk 1 \
+  --speculative-num-draft-tokens 4
+```
+
+- Transformers
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "tiny-random/glm-moe-dsa"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+input_ids = torch.randint(1000, 2000, size=(1, 2333), dtype=torch.long).cuda()  # trigger DSA
+# messages = [{"role": "user", "content": "hello"}]
+# input_ids = tokenizer(messages, return_tensors="pt").input_ids.cuda()
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    dtype=torch.bfloat16,
+    device_map="cuda",
+)
+generated_ids = model.generate(input_ids, max_new_tokens=32)
+output_text = tokenizer.decode(generated_ids[0][input_ids.shape[1]:])
+print(output_text)
+```
+
+### Codes to create this repo:
+
+<details>
+<summary>Click to expand</summary>
+
+```python
+import json
+from copy import deepcopy
+from pathlib import Path
+
+import accelerate
+import torch
+import torch.nn as nn
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "zai-org/GLM-5"
+save_folder = "/tmp/tiny-random/glm-moe-dsa"
+
+processor = AutoProcessor.from_pretrained(
+    source_model_id, trust_remote_code=True)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json: dict = json.load(f)
+
+head_dim = 64
+kv_lora_rank = 512
+qk_nope_head_dim = 192
+config_json.update({
+    "first_k_dense_replace": 1,
+    "mlp_layer_types": ['dense'] + ['sparse'],
+    "head_dim": head_dim,
+    "hidden_size": 8,
+    "index_head_dim": 32,
+    "index_n_heads": 4,
+    "intermediate_size": 32,
+    "moe_intermediate_size": 32,
+    "num_hidden_layers": 2,
+    'kv_lora_rank': kv_lora_rank,
+    "num_attention_heads": 4,
+    'num_key_value_heads': 4,
+    'q_lora_rank': 32,
+    "qk_head_dim": qk_nope_head_dim + head_dim,
+    'qk_nope_head_dim': qk_nope_head_dim,
+    'qk_rope_head_dim': head_dim,
+    'v_head_dim': qk_nope_head_dim + head_dim,
+    "tie_word_embeddings": True,
+})
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config)
+torch.set_default_dtype(torch.float32)
+
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+    model.generation_config.do_sample = True
+    print(model.generation_config)
+
+model = model.cpu()
+set_seed(42)
+n_params = sum(p.numel() for p in model.parameters())
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.1)
+        print(name, p.shape, p.numel() / n_params * 100, '%')
+# MTP
+set_seed(42)
+model.model.layers.append(nn.ModuleDict(dict(
+    shared_head=nn.ModuleDict(dict(
+        norm=nn.RMSNorm(config.hidden_size),
+        # head=deepcopy(model.model.embed_tokens),
+    )),
+    # embed_tokens=deepcopy(model.model.embed_tokens),
+    eh_proj=nn.Linear(config.hidden_size * 2,
+                      config.hidden_size, bias=False),
+    enorm=nn.RMSNorm(config.hidden_size),
+    hnorm=nn.RMSNorm(config.hidden_size),
+    input_layernorm=nn.RMSNorm(config.hidden_size),
+    post_attention_layernorm=nn.RMSNorm(config.hidden_size),
+    self_attn=deepcopy(model.model.layers[1].self_attn),
+    mlp=deepcopy(model.model.layers[1].mlp),
+)))
+for i in range(1, len(model.model.layers)):
+    model.model.layers[i].mlp.gate.e_score_correction_bias = torch.rand_like(
+        model.model.layers[i].mlp.gate.e_score_correction_bias).float()
+model.save_pretrained(save_folder)
+print(model)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+GlmMoeDsaForCausalLM(
+  (model): GlmMoeDsaModel(
+    (embed_tokens): Embedding(154880, 8, padding_idx=154820)
+    (layers): ModuleList(
+      (0): GlmMoeDsaDecoderLayer(
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=1792, bias=False)
+          (o_proj): Linear(in_features=1024, out_features=8, bias=False)
+          (wq_b): Linear(in_features=32, out_features=1024, bias=False)
+          (wk): Linear(in_features=8, out_features=256, bias=False)
+          (k_norm): GlmMoeDsaRMSNorm((256,), eps=1e-06)
+          (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+        )
+        (mlp): GlmMoeDsaMLP(
+          (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+          (up_proj): Linear(in_features=8, out_features=32, bias=False)
+          (down_proj): Linear(in_features=32, out_features=8, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+      )
+      (1): GlmMoeDsaDecoderLayer(
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=1792, bias=False)
+          (o_proj): Linear(in_features=1024, out_features=8, bias=False)
+          (wq_b): Linear(in_features=32, out_features=1024, bias=False)
+          (wk): Linear(in_features=8, out_features=256, bias=False)
+          (k_norm): GlmMoeDsaRMSNorm((256,), eps=1e-06)
+          (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+        )
+        (mlp): GlmMoeDsaMoE(
+          (experts): GlmMoeDsaNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): GlmMoeDsaTopkRouter()
+          (shared_experts): GlmMoeDsaMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+      )
+      (2): ModuleDict(
+        (shared_head): ModuleDict(
+          (norm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        )
+        (eh_proj): Linear(in_features=16, out_features=8, bias=False)
+        (enorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (hnorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (input_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (post_attention_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=1024, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=1792, bias=False)
+          (o_proj): Linear(in_features=1024, out_features=8, bias=False)
+          (wq_b): Linear(in_features=32, out_features=1024, bias=False)
+          (wk): Linear(in_features=8, out_features=256, bias=False)
+          (k_norm): GlmMoeDsaRMSNorm((256,), eps=1e-06)
+          (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+        )
+        (mlp): GlmMoeDsaMoE(
+          (experts): GlmMoeDsaNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): GlmMoeDsaTopkRouter()
+          (shared_experts): GlmMoeDsaMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+      )
+    )
+    (norm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+    (rotary_emb): GlmMoeDsaRotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=8, out_features=154880, bias=False)
+)
+```
+
+</details>

chat_template.jinja

@@ -0,0 +1,86 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{{ tool | tojson(ensure_ascii=False) }}
+{% endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1) %}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {% set ns.last_user_index = loop.index0 -%}
+    {%- endif %}
+{%- endfor %}
+{% for m in messages %}
+{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set reasoning_content = '' %}
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- else %}
+    {%- if '</think>' in content %}
+        {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+        {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+    {%- endif %}
+{%- endif %}
+{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
+{{ '<think>' + reasoning_content.strip() +  '</think>'}}
+{%- else -%}
+{{ '</think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{- '<tool_call>' + tc.name -}}
+{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if m.content is string -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' }}
+{%- endif %}
+{{- '<tool_response>' }}
+{{- m.content }}
+{{- '</tool_response>' }}
+{%- else -%}
+<|observation|>{% for tr in m.content %}
+<tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,64 @@
+{
+  "architectures": [
+    "GlmMoeDsaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "ep_size": 1,
+  "first_k_dense_replace": 1,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 8,
+  "index_head_dim": 32,
+  "index_n_heads": 4,
+  "index_topk": 2048,
+  "indexer_rope_interleave": true,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 202752,
+  "mlp_layer_types": [
+    "dense",
+    "sparse"
+  ],
+  "model_type": "glm_moe_dsa",
+  "moe_intermediate_size": 32,
+  "moe_layer_freq": 1,
+  "n_group": 1,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 4,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "num_nextn_predict_layers": 1,
+  "pad_token_id": 154820,
+  "pretraining_tp": 1,
+  "q_lora_rank": 32,
+  "qk_head_dim": 256,
+  "qk_nope_head_dim": 192,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-05,
+  "rope_interleave": true,
+  "rope_parameters": {
+    "rope_theta": 1000000,
+    "rope_type": "default"
+  },
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "tie_word_embeddings": true,
+  "topk_group": 1,
+  "topk_method": "noaux_tc",
+  "transformers_version": "5.2.0.dev0",
+  "use_cache": true,
+  "v_head_dim": 256,
+  "vocab_size": 154880
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "_from_model_config": true,
+  "do_sample": true,
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "pad_token_id": 154820,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "transformers_version": "5.2.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ed734a563b043f3746f8922c9447c582f2084b63369e5a0a75347e0b210a90a
+size 9455168

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d
+size 20217442

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|begin_of_transcription|>",
+    "<|end_of_transcription|>"
+  ],
+  "is_local": false,
+  "model_max_length": 202752,
+  "model_specific_special_tokens": {},
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "TokenizersBackend"
+}