Add files using upload-large-folder tool

.eval_results/lhtb.yaml

@@ -0,0 +1,9 @@
+- dataset:
+    id: IntelligenceLab/Long-Horizon-Terminal-Bench
+    task_id: lhtb
+  value: 38.5
+  date: '2026-07-16'
+  source:
+    url: https://zli12321.github.io/LHTB/leaderboard.html
+    name: LHTB leaderboard
+  notes: "mean reward x100 over 46 tasks (partial credit); [email protected]=3/46; official LHTB Harbor harness"

.eval_results/minimax-m3.yaml

@@ -0,0 +1,94 @@
+# Evaluation results for MiniMaxAI/MiniMax-M3
+# Extracted from the model card benchmark graph (figures/benchmark.jpeg)
+# https://huggingface.co/MiniMaxAI/MiniMax-M3
+# Paper: https://arxiv.org/abs/2606.13392
+
+# ---------------------------------------------------------------------------
+# Coding
+# ---------------------------------------------------------------------------
+
+# SWE-Bench Verified - 80.5
+- dataset:
+    id: SWE-bench/SWE-bench_Verified
+    task_id: swe_bench_%_resolved
+  value: 80.5
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Evaluated on internal infrastructure using Claude Code as the scaffolding. Each test was run 4 times and the average was taken."
+
+# SWE-Bench Pro - 59.0
+- dataset:
+    id: ScaleAI/SWE-bench_Pro
+    task_id: SWE_Bench_Pro
+  value: 59.0
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Evaluated on internal infrastructure using Claude Code as the scaffolding. Testing logic is aligned with the official evaluation."
+
+
+# ---------------------------------------------------------------------------
+# Multimodal
+# ---------------------------------------------------------------------------
+
+# MMMU-Pro - 78.1
+# MMMU-Pro defines three tasks: mmmu_pro_vision, mmmu_pro_standard_4_options,
+# mmmu_pro_standard_10_options. The model card reports a single "MMMU-Pro"
+# score without specifying the exact variant. We map it to the standard
+# 10-options task as the most common updated benchmark configuration.
+- dataset:
+    id: MMMU/MMMU_Pro
+    task_id: mmmu_pro_standard_10_options
+  value: 78.1
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "MMMU-Pro score extracted from the model card benchmark graph. The exact task variant (vision, standard 4-options, or standard 10-options) is not explicitly stated."
+
+# Video-MME (w/ sub) - 85.4
+# Mapped to Video-MME-v2, the registered successor benchmark on the Hub.
+- dataset:
+    id: MME-Benchmarks/Video-MME-v2
+    task_id: video-mme-v2
+  value: 85.4
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Model card reports 'VideoMME (w/ sub)'. Mapped to the closest registered benchmark on the Hub, Video-MME-v2."
+
+# ---------------------------------------------------------------------------
+# Cowork
+# ---------------------------------------------------------------------------
+
+# Claw-Eval - 74.5
+# Claw-Eval defines three tasks: general, multimodal, multi_turn. The model card
+# reports a single overall score, so it is mapped to the 'general' task.
+- dataset:
+    id: claw-eval/Claw-Eval
+    task_id: general
+  value: 74.5
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Model card reports a single 'Claw-Eval' score. Mapped to the 'general' task (overall); the exact task split is not specified."
+
+# Apex-Agents - 27.7
+- dataset:
+    id: mercor/apex-agents
+    task_id: apex-agents
+  value: 27.7
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Evaluated on the apex-agents benchmark."
+
+# YC-Bench - 2.1M (final assets in fund, monetary metric)
+- dataset:
+    id: collinear-ai/yc-bench
+    task_id: medium
+  value: 2100000
+  source:
+    url: https://huggingface.co/MiniMaxAI/MiniMax-M3
+    name: MiniMax-M3 model card
+  notes: "Model card reports 2.1M (monetary value, final assets fund). The benchmark's 'medium' task is used as the overall evaluation. Metric is monetary, not percentage-based."

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+figures/benchmark.jpeg filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,144 @@
+---
+base_model: MiniMaxAI/MiniMax-M3
+library_name: mlx
+pipeline_tag: text-generation
+tags:
+- mlx
+- minimax
+- mixture-of-experts
+- mixed-precision
+- quantization
+- thundermlx
+license: other
+license_name: minimax
+---
+
+# MiniMax-M3 Mixed-4.5bit MLX — the anti-overthinking quant
+
+A **mixed-precision MLX quantization of MiniMax-M3** (428B parameters, 23B
+active) that puts precision where decisions are made instead of spreading it
+evenly. Built for and served by
+[ThunderMLX](https://github.com/jonathan308/ThunderMLX), a 2-Mac pipeline
+serving stack for Apple Silicon.
+
+**TL;DR:** at +45 GB over the standard flat 4-bit (270 vs 225 GB), this quant
+closes ~28% of the entire fidelity gap to the bf16 model, cuts reasoning-loop
+"doom spirals" by 42–60%, eliminates 92% of hesitation markers, ships complete
+agentic artifacts instead of drafting them inside thinking — and finishes real
+tasks **15% faster in wall time** despite ~12% slower raw decode, because it
+stops second-guessing itself.
+
+## Why: flat 4-bit quantization causes overthinking
+
+Running MiniMax-M3 4-bit in agentic use, we kept hitting a failure family:
+thinking spirals that re-analyze the same paragraph with mutating wording,
+hesitation cascades ("wait… actually… let me reconsider"), and a stubborn
+habit of drafting entire code artifacts inside the thinking block while
+ignoring steering. Following arXiv 2606.00206 (quantization inflates
+hesitation-marker probabilities at high-entropy positions), we first shipped a
+runtime logit-penalty guard — it helped, but treated the symptom.
+
+The cause turned out to be *where* flat quantization spends its error budget.
+Rounding noise in a handful of small, decision-critical modules flips discrete
+choices: which experts fire, which KV blocks sparse attention reads, and which
+token wins the final logit race. This quant fixes those modules directly.
+
+## The recipe
+
+| Tier | Modules | Precision | Rationale |
+|---|---|---|---|
+| Decision | lm_head, all 57 MoE router gates, sparse-attention indexer projections | **8-bit / g64** | rounding noise here flips discrete choices — the literal overthinking mechanism |
+| Every-token | embeddings, all attention projections, dense-MLP layers | **6-bit / g64** | error compounds across all 60 layers with no routing dilution |
+| Bulk | all 129-expert fused MoE tensors | **4-bit / g32** | halved group size halves in-group rounding error; the cheapest quality lever on 96% of the weights |
+| Native | vision tower, norms (bf16), e_score_correction_bias (f32) | untouched | matches upstream |
+
+Effective average: ~4.8 bits/weight. Identical tensor names and MLX affine
+format to the standard 4-bit conversion — **loads anywhere the flat 4-bit
+loads**, no code changes.
+
+## Benchmarks
+
+### Distribution fidelity (teacher-forced EAR vs a bf16-grade reference, ~10k positions)
+
+EAR = per-position overlap between the quant's and the reference model's
+next-token distributions (metric from arXiv 2605.02404), normalized, higher
+is better. Reference = the bf16 checkpoint itself (experts at lossless 8-bit),
+evaluated with a layer-streaming pass.
+
+| Quant | Size | EAR mean | Worst-5% positions |
+|---|---|---|---|
+| flat 4-bit / g64 | 225 GB | 0.8747 | 0.5236 |
+| same-budget control (extra bits spread across bulk experts) | 268 GB | 0.8806 | 0.5493 |
+| **this quant** | 270 GB | **0.9103** | **0.6656** |
+
+The control experiment is the point: an equal-size quant that spends its extra
+bits on bulk experts recovers ~5% of the gap to bf16. Spending the same bits
+on the decision path recovers **~28%** — and **~30% at the hard-position tail**
+where reasoning behavior lives. Where the bits go matters far more than how
+many.
+
+### Behavior (identical prompts and seeds vs flat 4-bit, guard disabled)
+
+| Suite | flat 4-bit | this quant |
+|---|---|---|
+| Graded tasks — accuracy | 100% | **100%** |
+| Graded — avg thinking tokens | 176 | **121 (−31%)** |
+| Graded — hesitation markers/run | 0.60 | **0.05 (−92%)** |
+| Graded — avg wall time | 8.0 s | **6.8 s (−15%)** |
+| Loop probes (3 seeds) — avg thinking tokens | 1992 | **1159 (−42%)** |
+| Loop probes — hesitation markers | 28.9 | **7.7 (−73%)** |
+
+Ungoverned, this quant out-behaves the flat 4-bit running its most aggressive
+anti-overthinking logit penalty. On the flagship two-turn agentic test (build
+a complete single-file game, then steer), it plans in ~1k characters of
+thinking and ships a complete 46.8k-character working artifact in the answer —
+the flat 4-bit drafted the entire artifact inside its thinking block and
+resisted steering. Long thinking is preserved where it's warranted: hard
+constraint-solving still gets ~4k tokens of *forward-moving* reasoning
+(2.3% repeated-phrase churn vs >10% in true spirals).
+
+### Speed (2-Mac ThunderMLX pipeline, Thunderbolt RDMA, 38/22 layer split)
+
+| Metric | flat 4-bit | this quant |
+|---|---|---|
+| Decode, short context | ~28 tok/s | 23–26 tok/s |
+| Decode @ 70k context | ~27–29 tok/s | 23.8 tok/s (no depth collapse) |
+| Prefill @ 70k | — | 342 tok/s |
+| TTFT (warm) | ~1.4 s | ~1.4 s (unchanged) |
+
+The ~12% decode tax is repaid with interest on real tasks by shorter,
+non-redundant thinking (see wall times above).
+
+## Serving
+
+Built for [ThunderMLX](https://github.com/jonathan308/ThunderMLX) across two
+Apple Silicon Macs (tested: Mac Studio + MacBook Pro, 38/22 pipeline split,
+~187 GB + ~96 GB wired). Any MLX stack that serves the standard 4-bit
+conversion can load this model unchanged — same tensor names, same config
+schema, per-path quantization overrides declared in `config.json`.
+
+## Reproduce / adapt
+
+The converter, verification suite, and EAR evaluator are open source in the
+ThunderMLX repo (`ops/quant/`):
+
+- `m3_mixed_quant.py` — streaming mixed-precision converter: plan pass with a
+  name-set parity gate, per-expert rebuild of fused MoE tensors, incremental
+  5 GB shards, ~15 GB peak memory while converting an 854 GB checkpoint.
+- `ear_eval.py` / `ear_compare.py` — layer-streaming EAR evaluator: exact
+  next-token distributions from models far larger than RAM, including the
+  bf16 reference itself.
+
+Two upstream findings the tooling works around, relevant to anyone quantizing
+very large MoE models with MLX: (1) kernels evaluated on tensors above ~2³¹
+elements can silently corrupt output — fused MoE expert tensors are exactly
+that size, so the converter rebuilds them per-expert; (2) GPU kernels fed
+directly from memory-mapped files on slow external drives stall past the
+Metal watchdog — the converter materializes on the CPU stream first.
+
+## Acknowledgements
+
+- MiniMax for MiniMax-M3.
+- arXiv 2606.00206 (quantization-induced overthinking) for the mechanism, and
+  arXiv 2605.02404 (statistically-lossless quantization) for the EAR metric.
+- The MLX team — this entire pipeline runs on MLX.

added_tokens.json

@@ -0,0 +1,63 @@
+{
+  "]!p~[": 200000,
+  "<fim_prefix>": 200001,
+  "<fim_middle>": 200002,
+  "<fim_suffix>": 200003,
+  "<fim_pad>": 200004,
+  "<reponame>": 200005,
+  "<filename>": 200006,
+  "<gh_stars>": 200007,
+  "<issue_start>": 200008,
+  "<issue_comment>": 200009,
+  "<issue_closed>": 200010,
+  "<jupyter_start>": 200011,
+  "<jupyter_text>": 200012,
+  "<jupyter_code>": 200013,
+  "<jupyter_output>": 200014,
+  "<empty_output>": 200015,
+  "<commit_before>": 200016,
+  "<commit_msg>": 200017,
+  "<commit_after>": 200018,
+  "]~b]": 200019,
+  "[e~[": 200020,
+  "]!d~[": 200021,
+  "<function_call>": 200022,
+  "<code_interpreter>": 200023,
+  "]<]speech[>[": 200024,
+  "]<]image[>[": 200025,
+  "]<]video[>[": 200026,
+  "]<]start of speech[>[": 200027,
+  "]<]end of speech[>[": 200028,
+  "]<]start of image[>[": 200029,
+  "]<]end of image[>[": 200030,
+  "]<]start of video[>[": 200031,
+  "]<]end of video[>[": 200032,
+  "]<]vision pad[>[": 200033,
+  "]~!b[": 200034,
+  "<jupyter_error>": 200035,
+  "<add_file>": 200036,
+  "<delete_file>": 200037,
+  "<rename_file>": 200038,
+  "<edit_file>": 200039,
+  "<commit_message>": 200040,
+  "<empty_source_file>": 200041,
+  "<repo_struct>": 200042,
+  "<code_context>": 200043,
+  "<file_content>": 200044,
+  "<source_files>": 200045,
+  "<pr_start>": 200046,
+  "<review_comment>": 200047,
+  "<filepath>": 200048,
+  "<file_sep>": 200049,
+  "<think>": 200050,
+  "</think>": 200051,
+  "<tool_call>": 200052,
+  "</tool_call>": 200053,
+  "]<]frame[>[": 200054,
+  "]<]start of frame[>[": 200055,
+  "]<]end of frame[>[": 200056,
+  "<|content_altered_placeholder|>": 200057,
+  "]<]minimax[>[": 200058,
+  "<mm:think>": 200059,
+  "</mm:think>": 200060
+}

chat_template.jinja

@@ -0,0 +1,247 @@
+{# ---------- special token variables ---------- #}
+{%- set ns_token               = ']<]minimax[>['                  -%}
+{%- set bod_token              = ']~!b['                          -%}
+{%- set bos_token              = ']~b]'                           -%}
+{%- set eos_token              = '[e~['                           -%}
+{%- set toolcall_begin_token   = ns_token ~ '<tool_call>'         -%}
+{%- set toolcall_end_token     = ns_token ~ '</tool_call>'        -%}
+{%- set think_begin_token      = '<mm:think>'                     -%}
+{%- set think_end_token        = '</mm:think>'                    -%}
+{%- set image_token            = ']<]image[>['                    -%}
+{%- set video_token            = ']<]video[>['                    -%}
+{#- Thinking mode: "enabled" / "disabled" / "adaptive" / not defined -#}
+{#- Recursive XML renderer for tool_call arguments ======================== -#}
+{#- None values are intentionally skipped in mapping iteration so that
+    `<key>null</key>` (which would round-trip to the literal string "null")
+    never appears in the rendered tool_call. The convention is: omit the
+    field entirely. The top-level `_args` loop applies the same rule.
+    The `val is none` branch below is a safety net only — upstream cleaning
+    (drop_none_in_tool_arguments) should ensure no None ever reaches here. -#}
+{%- macro to_xml(val, ns) -%}
+{%- if val is mapping -%}
+{%- for k, v in val.items() if v is not none -%}
+{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>
+{%- endfor -%}
+{%- elif val is iterable and val is not string -%}
+{%- for item in val -%}
+{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>
+{%- endfor -%}
+{%- elif val is none -%}
+{#- Should be unreachable when upstream cleaning is applied. -#}
+{%- elif val is boolean -%}
+{{ val | tojson }}
+{%- else -%}
+{{ val }}
+{%- endif -%}
+{%- endmacro -%}
+{#- Tool Rendering Functions ============================================== -#}
+{%- macro render_tool_namespace(namespace_name, tool_list) -%}
+{%- for tool in tool_list -%}
+<tool>{{ tool.function | tojson(ensure_ascii=False) }}</tool>
+{% endfor -%}
+{%- endmacro -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{ content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is mapping and item.type == 'image' -%}
+                {{- image_token }}
+            {%- elif item is mapping and item.type == 'video' -%}
+                {{- video_token}}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is none -%}
+        {{- '' }}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{#- System Message Construction ============================================ -#}
+{%- macro build_system_message(system_message) -%}
+    {%- if system_message and system_message.content -%}
+        {{- visible_text(system_message.content) }}
+    {%- else -%}
+        {{- 'Your model version is MiniMax-M3, developed by MiniMax. Knowledge cutoff: January 2026. Founded in early 2022, MiniMax is a global AI foundation model company committed to advancing the frontiers of AI towards AGI.' }}
+    {%- endif -%}
+
+    {#- Thinking mode instructions -#}
+    {{- '\n\n<thinking_instructions>\n' }}
+    {{- 'You have a thinking capability that allows you to reason step by step before responding. When thinking is enabled, wrap your reasoning in ' ~ think_begin_token ~ think_end_token ~ ' tags before your response. When thinking is disabled, begin your response directly after the ' ~ think_end_token ~ ' prefix. When thinking is adaptive, decide on your own whether to think for the current turn.\n' }}
+    {%- if thinking_mode is defined -%}
+        {%- if thinking_mode == "enabled" -%}
+            {{- 'Current thinking mode: enabled. You MUST think step by step before every response, including after receiving function/tool results.\n' }}
+        {%- elif thinking_mode == "disabled" -%}
+            {{- 'Current thinking mode: disabled. Do not output any thinking process.\n' }}
+        {%- elif thinking_mode == "adaptive" -%}
+            {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\n' }}
+        {%- endif -%}
+    {%- else -%}
+        {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\n' }}
+    {%- endif -%}
+    {{- '</thinking_instructions>' }}
+{%- endmacro -%}
+{%- macro build_developer_message(developer_message) -%}
+    {%- if developer_message and developer_message.content -%}
+        {{- visible_text(developer_message.content) }}
+    {%- else -%}
+        {%- if model_identity is not defined -%}
+            {%- set model_identity = "You are a helpful assistant." -%}
+        {%- endif -%}
+        {{- model_identity }}
+    {%- endif -%}
+{%- endmacro -%}
+{#- Main Template Logic ================================================= -#}
+{#- Role mapping: root -> system sp (high priority), system/developer -> developer sp (low priority) -#}
+{%- set system_message = none -%}
+{%- set developer_message = none -%}
+{%- set conversation_messages = messages -%}
+{%- if messages and messages[0].role == "root" -%}
+    {%- set system_message = messages[0] -%}
+    {%- set conversation_messages = messages[1:] -%}
+    {%- if conversation_messages and conversation_messages[0].role in ["system", "developer"] -%}
+        {%- set developer_message = conversation_messages[0] -%}
+        {%- set conversation_messages = conversation_messages[1:] -%}
+    {%- endif -%}
+{%- elif messages and messages[0].role in ["system", "developer"] -%}
+    {%- set developer_message = messages[0] -%}
+    {%- set conversation_messages = messages[1:] -%}
+{%- endif -%}
+{#- Render system sp (higher priority, root role only) -#}
+{{- bod_token ~ bos_token ~ 'system' ~ '\n' }}
+{{- build_system_message(system_message) }}
+{{- eos_token ~ '\n' }}
+
+{#- Render developer sp (lower priority: system/developer role + tools) -#}
+{{- bos_token ~ 'developer' ~ '\n' }}
+{{- build_developer_message(developer_message) }}
+{%- if tools -%}
+    {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }}
+    {{- '\n' ~ '<tools>' ~ '\n' }}
+    {{- render_tool_namespace("functions", tools) }}
+    {{- '</tools>' ~ '\n\n' }}
+    {{- 'To call tools, wrap all invocations in a single ' ~ toolcall_begin_token ~ toolcall_end_token ~ ' block. Parameter values containing nested objects or arrays are recursively expanded into XML elements. Example:\n' }}
+    {{- '\n' ~ toolcall_begin_token ~ '\n' }}
+    {{- ns_token + '<invoke name="tool-name-1">' }}
+    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}
+    {{- ns_token + '<param-2>' }}
+    {{- ns_token + '<item>' }}
+    {{- ns_token + '<key-a>val-a' + ns_token + '</key-a>' }}
+    {{- ns_token + '<key-b>val-b' + ns_token + '</key-b>' }}
+    {{- ns_token + '</item>' }}
+    {{- ns_token + '</param-2>' }}
+    {{- ns_token + '</invoke>\n' }}
+    {{- ns_token + '<invoke name="tool-name-2">' }}
+    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}
+    {{- ns_token + '</invoke>\n' }}
+    {{- toolcall_end_token }}
+{%- endif -%}
+{{- eos_token ~ '\n' }}
+
+{#- Render messages -#}
+{%- set last_tool_call = namespace(name=none) -%}
+{%- for message in conversation_messages -%}
+    {%- if message.role == 'assistant' -%}
+        {{- bos_token ~ 'ai' ~ '\n' }}
+
+        {%- set reasoning_content = '' %}
+        {%- set content = visible_text(message.content) %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if think_end_token in content %}
+                {%- set reasoning_content = content.split(think_end_token)[0].strip('\n').split(think_begin_token)[-1].strip('\n') %}
+                {%- set content = content.split(think_end_token)[-1].strip('\n') %}
+            {%- endif %}
+        {%- endif %}
+
+        {%- if reasoning_content -%}
+            {#- Render thinking for every assistant turn (all-turn visible) -#}
+            {{- think_begin_token ~ reasoning_content ~ think_end_token }}
+        {%- else -%}
+            {#- No thinking rendered → prefix with think_end_token -#}
+            {{- think_end_token }}
+        {%- endif -%}
+
+        {%- if content -%}
+            {{- content }}
+        {%- endif -%}
+        {%- if message.tool_calls -%}
+            {{- toolcall_begin_token ~ '\n' }}
+
+            {%- for tool_call in message.tool_calls -%}
+                {%- if tool_call.function -%}
+                    {%- set tool_call = tool_call.function -%}
+                {%- endif -%}
+{{- ns_token + '<invoke name="' + tool_call.name + '">' }}
+{%- set _args = tool_call.arguments -%}
+{%- for k, v in _args.items() if v is not none %}
+{{- ns_token + '<' + k + '>' -}}
+{{- to_xml(v, ns_token) -}}
+{{- ns_token + '</' + k + '>' }}
+{%- endfor -%}
+{{- ns_token + '</invoke>' ~ '\n' }}
+            {%- endfor -%}
+
+            {{- toolcall_end_token }}
+            {%- if message.tool_calls[-1].function -%}
+                {%- set last_tool_call.name = message.tool_calls[-1].function.name -%}
+            {%- else -%}
+                {%- set last_tool_call.name = message.tool_calls[-1].name -%}
+            {%- endif -%}
+        {%- else -%}
+            {%- set last_tool_call.name = none -%}
+        {%- endif -%}
+        {{- eos_token ~ '\n' }}
+
+    {%- elif message.role == 'tool' -%}
+        {%- if last_tool_call.name is none -%}
+            {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }}
+        {%- endif -%}
+        {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%}
+            {{- bos_token ~ 'tool' }}
+        {%- endif -%}
+        {{- '\n<response>' }}
+        {%- if message.content is string -%}
+            {{- message.content }}
+        {%- else -%}
+            {%- for tr in message.content -%}
+                {%- if tr is mapping and tr.type is defined and tr.type == 'image' -%}
+                    {{- image_token }}
+                {%- elif tr is mapping and tr.type is defined and tr.type == 'video' -%}
+                    {{- video_token }}
+                {%- else -%}
+                    {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }}
+                {%- endif -%}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- '</response>' }}
+        {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%}
+            {{- eos_token ~ '\n' -}}
+        {%- endif -%}
+
+    {%- elif message.role == 'user' -%}
+        {{- bos_token ~ 'user' ~ '\n' }}
+        {{- visible_text(message.content) }}
+        {{- eos_token ~ '\n' }}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Generation prompt -#}
+{%- if add_generation_prompt -%}
+{{- bos_token ~ 'ai' ~ '\n' }}
+{%- if thinking_mode is defined and thinking_mode == "disabled" -%}
+    {{- think_end_token }}
+{%- elif thinking_mode is defined and thinking_mode == "adaptive" -%}
+    {#- adaptive: no prefix, let model decide -#}
+{%- elif thinking_mode is defined and thinking_mode == "enabled" -%}
+    {#- enabled or not defined: default to think -#}
+    {{- think_begin_token }}
+{%- else -%}
+    {#- adaptive: no prefix, let model decide -#}
+{%- endif -%}
+{%- endif -%}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{# ---------- special token variables ---------- #}\n{%- set ns_token               = ']<]minimax[>['                  -%}\n{%- set bod_token              = ']~!b['                          -%}\n{%- set bos_token              = ']~b]'                           -%}\n{%- set eos_token              = '[e~['                           -%}\n{%- set toolcall_begin_token   = ns_token ~ '<tool_call>'         -%}\n{%- set toolcall_end_token     = ns_token ~ '</tool_call>'        -%}\n{%- set think_begin_token      = '<mm:think>'                     -%}\n{%- set think_end_token        = '</mm:think>'                    -%}\n{%- set image_token            = ']<]image[>['                    -%}\n{%- set video_token            = ']<]video[>['                    -%}\n{#- Thinking mode: \"enabled\" / \"disabled\" / \"adaptive\" / not defined -#}\n{#- Recursive XML renderer for tool_call arguments ======================== -#}\n{#- None values are intentionally skipped in mapping iteration so that\n    `<key>null</key>` (which would round-trip to the literal string \"null\")\n    never appears in the rendered tool_call. The convention is: omit the\n    field entirely. The top-level `_args` loop applies the same rule.\n    The `val is none` branch below is a safety net only \u2014 upstream cleaning\n    (drop_none_in_tool_arguments) should ensure no None ever reaches here. -#}\n{%- macro to_xml(val, ns) -%}\n{%- if val is mapping -%}\n{%- for k, v in val.items() if v is not none -%}\n{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>\n{%- endfor -%}\n{%- elif val is iterable and val is not string -%}\n{%- for item in val -%}\n{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>\n{%- endfor -%}\n{%- elif val is none -%}\n{#- Should be unreachable when upstream cleaning is applied. -#}\n{%- elif val is boolean -%}\n{{ val | tojson }}\n{%- else -%}\n{{ val }}\n{%- endif -%}\n{%- endmacro -%}\n{#- Tool Rendering Functions ============================================== -#}\n{%- macro render_tool_namespace(namespace_name, tool_list) -%}\n{%- for tool in tool_list -%}\n<tool>{{ tool.function | tojson(ensure_ascii=False) }}</tool>\n{% endfor -%}\n{%- endmacro -%}\n{%- macro visible_text(content) -%}\n    {%- if content is string -%}\n        {{ content }}\n    {%- elif content is iterable and content is not mapping -%}\n        {%- for item in content -%}\n            {%- if item is mapping and item.type == 'text' -%}\n                {{- item.text }}\n            {%- elif item is mapping and item.type == 'image' -%}\n                {{- image_token }}\n            {%- elif item is mapping and item.type == 'video' -%}\n                {{- video_token}}\n            {%- elif item is string -%}\n                {{- item }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- elif content is none -%}\n        {{- '' }}\n    {%- else -%}\n        {{- content }}\n    {%- endif -%}\n{%- endmacro -%}\n{#- System Message Construction ============================================ -#}\n{%- macro build_system_message(system_message) -%}\n    {%- if system_message and system_message.content -%}\n        {{- visible_text(system_message.content) }}\n    {%- else -%}\n        {{- 'Your model version is MiniMax-M3, developed by MiniMax. Knowledge cutoff: January 2026. Founded in early 2022, MiniMax is a global AI foundation model company committed to advancing the frontiers of AI towards AGI.' }}\n    {%- endif -%}\n\n    {#- Thinking mode instructions -#}\n    {{- '\\n\\n<thinking_instructions>\\n' }}\n    {{- 'You have a thinking capability that allows you to reason step by step before responding. When thinking is enabled, wrap your reasoning in ' ~ think_begin_token ~ think_end_token ~ ' tags before your response. When thinking is disabled, begin your response directly after the ' ~ think_end_token ~ ' prefix. When thinking is adaptive, decide on your own whether to think for the current turn.\\n' }}\n    {%- if thinking_mode is defined -%}\n        {%- if thinking_mode == \"enabled\" -%}\n            {{- 'Current thinking mode: enabled. You MUST think step by step before every response, including after receiving function/tool results.\\n' }}\n        {%- elif thinking_mode == \"disabled\" -%}\n            {{- 'Current thinking mode: disabled. Do not output any thinking process.\\n' }}\n        {%- elif thinking_mode == \"adaptive\" -%}\n            {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\\n' }}\n        {%- endif -%}\n    {%- else -%}\n        {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\\n' }}\n    {%- endif -%}\n    {{- '</thinking_instructions>' }}\n{%- endmacro -%}\n{%- macro build_developer_message(developer_message) -%}\n    {%- if developer_message and developer_message.content -%}\n        {{- visible_text(developer_message.content) }}\n    {%- else -%}\n        {%- if model_identity is not defined -%}\n            {%- set model_identity = \"You are a helpful assistant.\" -%}\n        {%- endif -%}\n        {{- model_identity }}\n    {%- endif -%}\n{%- endmacro -%}\n{#- Main Template Logic ================================================= -#}\n{#- Role mapping: root -> system sp (high priority), system/developer -> developer sp (low priority) -#}\n{%- set system_message = none -%}\n{%- set developer_message = none -%}\n{%- set conversation_messages = messages -%}\n{%- if messages and messages[0].role == \"root\" -%}\n    {%- set system_message = messages[0] -%}\n    {%- set conversation_messages = messages[1:] -%}\n    {%- if conversation_messages and conversation_messages[0].role in [\"system\", \"developer\"] -%}\n        {%- set developer_message = conversation_messages[0] -%}\n        {%- set conversation_messages = conversation_messages[1:] -%}\n    {%- endif -%}\n{%- elif messages and messages[0].role in [\"system\", \"developer\"] -%}\n    {%- set developer_message = messages[0] -%}\n    {%- set conversation_messages = messages[1:] -%}\n{%- endif -%}\n{#- Render system sp (higher priority, root role only) -#}\n{{- bod_token ~ bos_token ~ 'system' ~ '\\n' }}\n{{- build_system_message(system_message) }}\n{{- eos_token ~ '\\n' }}\n\n{#- Render developer sp (lower priority: system/developer role + tools) -#}\n{{- bos_token ~ 'developer' ~ '\\n' }}\n{{- build_developer_message(developer_message) }}\n{%- if tools -%}\n    {{- '\\n\\n' ~ '# Tools' ~ '\\n' ~ 'You may call one or more tools to assist with the user query.\\nHere are the tools available in JSONSchema format:' ~ '\\n' }}\n    {{- '\\n' ~ '<tools>' ~ '\\n' }}\n    {{- render_tool_namespace(\"functions\", tools) }}\n    {{- '</tools>' ~ '\\n\\n' }}\n    {{- 'To call tools, wrap all invocations in a single ' ~ toolcall_begin_token ~ toolcall_end_token ~ ' block. Parameter values containing nested objects or arrays are recursively expanded into XML elements. Example:\\n' }}\n    {{- '\\n' ~ toolcall_begin_token ~ '\\n' }}\n    {{- ns_token + '<invoke name=\"tool-name-1\">' }}\n    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}\n    {{- ns_token + '<param-2>' }}\n    {{- ns_token + '<item>' }}\n    {{- ns_token + '<key-a>val-a' + ns_token + '</key-a>' }}\n    {{- ns_token + '<key-b>val-b' + ns_token + '</key-b>' }}\n    {{- ns_token + '</item>' }}\n    {{- ns_token + '</param-2>' }}\n    {{- ns_token + '</invoke>\\n' }}\n    {{- ns_token + '<invoke name=\"tool-name-2\">' }}\n    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}\n    {{- ns_token + '</invoke>\\n' }}\n    {{- toolcall_end_token }}\n{%- endif -%}\n{{- eos_token ~ '\\n' }}\n\n{#- Render messages -#}\n{%- set last_tool_call = namespace(name=none) -%}\n{%- for message in conversation_messages -%}\n    {%- if message.role == 'assistant' -%}\n        {{- bos_token ~ 'ai' ~ '\\n' }}\n\n        {%- set reasoning_content = '' %}\n        {%- set content = visible_text(message.content) %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if think_end_token in content %}\n                {%- set reasoning_content = content.split(think_end_token)[0].strip('\\n').split(think_begin_token)[-1].strip('\\n') %}\n                {%- set content = content.split(think_end_token)[-1].strip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n\n        {%- if reasoning_content -%}\n            {#- Render thinking for every assistant turn (all-turn visible) -#}\n            {{- think_begin_token ~ reasoning_content ~ think_end_token }}\n        {%- else -%}\n            {#- No thinking rendered \u2192 prefix with think_end_token -#}\n            {{- think_end_token }}\n        {%- endif -%}\n\n        {%- if content -%}\n            {{- content }}\n        {%- endif -%}\n        {%- if message.tool_calls -%}\n            {{- toolcall_begin_token ~ '\\n' }}\n\n            {%- for tool_call in message.tool_calls -%}\n                {%- if tool_call.function -%}\n                    {%- set tool_call = tool_call.function -%}\n                {%- endif -%}\n{{- ns_token + '<invoke name=\"' + tool_call.name + '\">' }}\n{%- set _args = tool_call.arguments -%}\n{%- for k, v in _args.items() if v is not none %}\n{{- ns_token + '<' + k + '>' -}}\n{{- to_xml(v, ns_token) -}}\n{{- ns_token + '</' + k + '>' }}\n{%- endfor -%}\n{{- ns_token + '</invoke>' ~ '\\n' }}\n            {%- endfor -%}\n\n            {{- toolcall_end_token }}\n            {%- if message.tool_calls[-1].function -%}\n                {%- set last_tool_call.name = message.tool_calls[-1].function.name -%}\n            {%- else -%}\n                {%- set last_tool_call.name = message.tool_calls[-1].name -%}\n            {%- endif -%}\n        {%- else -%}\n            {%- set last_tool_call.name = none -%}\n        {%- endif -%}\n        {{- eos_token ~ '\\n' }}\n\n    {%- elif message.role == 'tool' -%}\n        {%- if last_tool_call.name is none -%}\n            {{- raise_exception(\"Message has tool role, but there was no previous assistant message with a tool call!\") }}\n        {%- endif -%}\n        {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%}\n            {{- bos_token ~ 'tool' }}\n        {%- endif -%}\n        {{- '\\n<response>' }}\n        {%- if message.content is string -%}\n            {{- message.content }}\n        {%- else -%}\n            {%- for tr in message.content -%}\n                {%- if tr is mapping and tr.type is defined and tr.type == 'image' -%}\n                    {{- image_token }}\n                {%- elif tr is mapping and tr.type is defined and tr.type == 'video' -%}\n                    {{- video_token }}\n                {%- else -%}\n                    {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }}\n                {%- endif -%}\n            {%- endfor -%}\n        {%- endif -%}\n        {{- '</response>' }}\n        {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%}\n            {{- eos_token ~ '\\n' -}}\n        {%- endif -%}\n\n    {%- elif message.role == 'user' -%}\n        {{- bos_token ~ 'user' ~ '\\n' }}\n        {{- visible_text(message.content) }}\n        {{- eos_token ~ '\\n' }}\n    {%- endif -%}\n{%- endfor -%}\n\n{#- Generation prompt -#}\n{%- if add_generation_prompt -%}\n{{- bos_token ~ 'ai' ~ '\\n' }}\n{%- if thinking_mode is defined and thinking_mode == \"disabled\" -%}\n    {{- think_end_token }}\n{%- elif thinking_mode is defined and thinking_mode == \"adaptive\" -%}\n    {#- adaptive: no prefix, let model decide -#}\n{%- elif thinking_mode is defined and thinking_mode == \"enabled\" -%}\n    {#- enabled or not defined: default to think -#}\n    {{- think_begin_token }}\n{%- else -%}\n    {#- adaptive: no prefix, let model decide -#}\n{%- endif -%}\n{%- endif -%}\n"
+}

configuration_minimax_m3_vl.py

@@ -0,0 +1,111 @@
+"""HuggingFace configs for the MiniMax VL family (M2 VL / M3 VL).
+
+This file is bundled into every converted HF checkpoint so that loading via
+``AutoConfig.from_pretrained(..., trust_remote_code=True)`` works without any
+runtime dependency on sglang or other internal packages — only stock
+``transformers`` is required.
+
+The class definitions intentionally mirror
+``sglang.srt.configs.minimax_vl``; if either side changes, keep them in sync.
+
+The file is named ``configuration_minimax_m3_vl.py`` (matching the legacy
+``model_type="minimax_m3_vl"`` and the converter's ``auto_map`` entry) so
+that ckpts produced by this converter remain loadable by older sglang versions
+that only know the ``MiniMaxM3VL*`` names. The canonical class is
+``MiniMaxM3VLConfig``; ``MiniMaxM3VLConfig`` is a thin BC alias whose only
+purpose is to be referenced from ``auto_map``.
+"""
+
+from typing import Optional
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.auto import CONFIG_MAPPING
+
+
+def _coerce_sub_config(
+    sub_config: Optional[dict], default_model_type: str
+) -> Optional[PretrainedConfig]:
+    """Convert a config dict to a ``PretrainedConfig`` instance.
+
+    If ``model_type`` is registered in HF ``CONFIG_MAPPING`` the corresponding
+    config class is used; otherwise we fall back to a generic
+    ``PretrainedConfig`` so all dict keys still become real attributes (M3's
+    text backbone uses ``model_type="minimax_m2"`` which is not in
+    ``CONFIG_MAPPING``).
+    """
+    if not isinstance(sub_config, dict):
+        return sub_config
+    model_type = sub_config.get("model_type", default_model_type)
+    cls = CONFIG_MAPPING.get(model_type, PretrainedConfig)
+    return cls(**sub_config)
+
+
+class MiniMaxVLBaseConfig(PretrainedConfig):
+    """Base config shared by every MiniMax VL variant.
+
+    Handles vision/text sub-config coercion. Concrete subclasses only need to
+    declare a unique ``model_type`` string.
+    """
+
+    def __init__(
+        self,
+        vision_config: Optional[dict] = None,
+        text_config: Optional[dict] = None,
+        image_token_index: int = 200025,
+        video_token_index: int = 200026,
+        image_seq_length: int = 576,
+        process_image_mode: str = "dynamic_res",
+        projector_hidden_act: str = "gelu",
+        multimodal_projector_bias: bool = True,
+        vision_feature_layer: int = -1,
+        vision_feature_select_strategy: str = "full",
+        img_token_compression_config: Optional[dict] = None,
+        image_grid_pinpoints: Optional[str] = None,
+        **kwargs,
+    ):
+        self.vision_config = _coerce_sub_config(vision_config, "clip_vision_model")
+        self.text_config = _coerce_sub_config(text_config, "mixtral")
+
+        self.image_token_index = image_token_index
+        self.video_token_index = video_token_index
+        self.image_seq_length = image_seq_length
+        self.process_image_mode = process_image_mode
+        self.projector_hidden_act = projector_hidden_act
+        self.multimodal_projector_bias = multimodal_projector_bias
+        self.vision_feature_layer = vision_feature_layer
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+        self.img_token_compression_config = img_token_compression_config or {}
+        self.image_grid_pinpoints = image_grid_pinpoints
+
+        super().__init__(**kwargs)
+
+    def __post_init__(self, **kwargs):
+        super().__post_init__(**kwargs)
+        if hasattr(self, "vision_config"):
+            self.vision_config = _coerce_sub_config(self.vision_config, "clip_vision_model")
+        if hasattr(self, "text_config"):
+            self.text_config = _coerce_sub_config(self.text_config, "mixtral")
+
+
+class MiniMaxM2VLConfig(MiniMaxVLBaseConfig):
+    """MiniMax M2 VL: vision tower + M2 (Mixtral-style MoE) text backbone."""
+
+    model_type = "minimax_m2_vl"
+
+
+class MiniMaxM3VLConfig(MiniMaxVLBaseConfig):
+    """MiniMax M3 VL: vision tower + M3 (mixed sparse/dense MoE) text backbone."""
+
+    model_type = "minimax_m3_vl"
+
+
+class MiniMaxM2MiniVLConfig(MiniMaxM2VLConfig):
+    """Legacy alias kept so old ``model_type="minimax_m2_mini_vl"`` ckpts load."""
+
+    model_type = "minimax_m2_mini_vl"
+
+
+class MiniMaxM3VLConfig(MiniMaxM3VLConfig):
+    """Legacy alias kept so old ``model_type="minimax_m3_vl"`` ckpts load."""
+
+    model_type = "minimax_m3_vl"

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "bos_token_id": 200019,
+  "do_sample": true,
+  "eos_token_id": 200020,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "transformers_version": "4.46.1"
+}

image_processor.py

@@ -0,0 +1,223 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+"""
+MiniMax VL family HuggingFace-compatible Processor, ImageProcessor, VideoProcessor.
+"""
+import math
+from typing import List, Tuple
+
+import torch
+from torchvision.transforms import InterpolationMode
+from transformers import BatchFeature
+from transformers.image_processing_utils_fast import (
+    BaseImageProcessorFast,
+    group_images_by_shape,
+    reorder_images,
+)
+from transformers.image_utils import PILImageResampling, SizeDict
+from transformers.processing_utils import (
+    ImagesKwargs,
+    Unpack,
+)
+from transformers.utils import TensorType
+
+MAX_RATIO = 200
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 4 * 28 * 28,
+    max_pixels: int = 451584,
+) -> tuple[int, int]:
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
+            f"got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+# ==============================================================================
+# MiniMax M3 VL Image Processor Fast (Fast Mode - Torch based)
+# ==============================================================================
+
+
+class MiniMaxM3VLImageProcessorKwargs(ImagesKwargs, total=False):
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+    max_pixels: int
+
+
+class MiniMaxM3VLImageProcessor(BaseImageProcessorFast):
+    do_resize = True
+    resample = PILImageResampling.BICUBIC
+    size = {"height": 672, "width": 672}  # required by base class validation, not used as resize bound
+    default_to_square = False
+    do_rescale = True
+    rescale_factor = 1 / 255
+    do_normalize = True
+    image_mean = [0.48145466, 0.4578275, 0.40821073]
+    image_std = [0.26862954, 0.26130258, 0.27577711]
+    do_convert_rgb = True
+    patch_size = 14
+    temporal_patch_size = 2
+    merge_size = 2
+    max_pixels = 451584             # 672*672
+    valid_kwargs = MiniMaxM3VLImageProcessorKwargs
+    model_input_names = ["pixel_values", "image_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    def preprocess(
+        self, images, **kwargs: Unpack[MiniMaxM3VLImageProcessorKwargs]
+    ) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def _preprocess(
+        self,
+        images: List[torch.Tensor],
+        do_resize: bool,
+        size: SizeDict,
+        resample: PILImageResampling | InterpolationMode | int | None,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | List[float] | None,
+        image_std: float | List[float] | None,
+        patch_size: int,
+        temporal_patch_size: int,
+        merge_size: int,
+        max_pixels: int,
+        disable_grouping: bool | None,
+        return_tensors: str | TensorType | None,
+        **kwargs,
+    ) -> BatchFeature:
+        grouped_images, grouped_images_index = group_images_by_shape(
+            images, disable_grouping=disable_grouping
+        )
+        resized_images_grouped = {}
+        factor = patch_size * merge_size
+        for shape, stacked_images in grouped_images.items():
+            height, width = stacked_images.shape[-2:]
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height, width, factor=factor,
+                    max_pixels=max_pixels,
+                )
+                stacked_images = self.resize(
+                    stacked_images,
+                    size=SizeDict(height=resized_height, width=resized_width),
+                    resample=resample,
+                )
+            resized_images_grouped[shape] = stacked_images
+
+        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
+
+        grouped_images, grouped_images_index = group_images_by_shape(
+            resized_images, disable_grouping=disable_grouping
+        )
+        processed_images_grouped = {}
+        processed_grids = {}
+
+        for shape, stacked_images in grouped_images.items():
+            resized_height, resized_width = stacked_images.shape[-2:]
+
+            patches = self.rescale_and_normalize(
+                stacked_images,
+                do_rescale,
+                rescale_factor,
+                do_normalize,
+                image_mean,
+                image_std,
+            )
+            if patches.ndim == 4:
+                patches = patches.unsqueeze(1)
+
+            if patches.shape[1] % temporal_patch_size != 0:
+                repeats = patches[:, -1:].repeat(
+                    1,
+                    temporal_patch_size - (patches.shape[1] % temporal_patch_size),
+                    1,
+                    1,
+                    1,
+                )
+                patches = torch.cat([patches, repeats], dim=1)
+
+            batch_size, grid_t, channel = patches.shape[:3]
+            grid_t = grid_t // temporal_patch_size
+            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+            patches = patches.view(
+                batch_size,
+                grid_t,
+                temporal_patch_size,
+                channel,
+                grid_h // merge_size,
+                merge_size,
+                patch_size,
+                grid_w // merge_size,
+                merge_size,
+                patch_size,
+            )
+            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+
+            flatten_patches = patches.reshape(
+                batch_size,
+                grid_t * grid_h * grid_w,
+                channel * temporal_patch_size * patch_size * patch_size,
+            )
+
+            processed_images_grouped[shape] = flatten_patches
+            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
+
+        processed_images = reorder_images(
+            processed_images_grouped, grouped_images_index
+        )
+        processed_grids = reorder_images(processed_grids, grouped_images_index)
+
+        pixel_values = torch.cat(processed_images, dim=0)
+        image_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
+
+        return BatchFeature(
+            data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw},
+            tensor_type=return_tensors,
+        )
+
+    def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None):
+        images_kwargs = images_kwargs or {}
+        patch_size = images_kwargs.get("patch_size", self.patch_size)
+        merge_size = images_kwargs.get("merge_size", self.merge_size)
+        max_pixels = images_kwargs.get("max_pixels", self.max_pixels)
+
+        resized_height, resized_width = smart_resize(
+            height, width, factor=patch_size * merge_size,
+            max_pixels=max_pixels,
+        )
+        grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+        return grid_h * grid_w

model-00001-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e2c66bb6e03673e396a88a207d031fee08208aa4c2e9879580db74fd7613d1
+size 3166017650

model-00003-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb9f05764648306a73a77425a71ddcbb1b8c1fe5058f666983bb92b6cd5bb04
+size 4657231227

model-00005-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e7f2cce601cad5c95c1dc0b11f35697e59bea7a35f16ede4b618ca8792f3967
+size 4657231159

model-00006-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0257598613aa5ebb3140f18274e574b8fe89710f0c769f1bf13ebe677e409c8
+size 4657231149

model-00009-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:080c2190da17149b01ee5a7878e57e387fd03e4349e38f4bfa1f10f4cabf1389
+size 4657231174

model-00011-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0f889d08d256ef5200a4ab82e629a52337cf82b525ef5f1c625aec419937890
+size 4657231192

model-00012-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38c03234662b9bfa094d1eff540608732324276d9f0706bc2bd632e6eff4528a
+size 4657231166

model-00017-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0a921ca2b0569a83e9506a0d2618274f8c1d36946338e7f9cbdee6b25766b43
+size 4657231236

model-00018-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8327c7db553d19ff6061f30a33ade74212b4951a194122fb96741eaa2fc248ae
+size 4657231160

model-00020-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da1b452c34be78661aecb4586c59f3742df23da3032b908c323c23311117f840
+size 4657231208

model-00024-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e25e065bc8aee93642d76f43355e8abd5c8e4f07d2190ce214a9e08023ab2f
+size 4657231230

model-00025-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:725727316742a7ea552e279a76decbfece07ef2dcce214a940f19a0c504e3e17
+size 4657231206

model-00029-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf3a79010f022ab039c3352d7daac917ed5ee95bbe7493b59613bcd9ee200b0d
+size 4657231202

model-00032-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b6dac609f239168323915693ddaa20d01d896c1323a5c6045a64d3244e4c856
+size 4657231216

model-00034-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fac4f81879cac17ce1ca421fcee51327710756127e7bf26816ab1df691adcea
+size 4657231234

model-00037-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6788eeae35f9c7295404f35b4164269a0b3d872886c4498baedb9ef0ab1d9904
+size 4657231188

model-00038-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5364d3065dc49ca296391ff9bc58e770bad53eaf19f46833400cb784946ad12
+size 4657231232

model-00041-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9779ee56b3ec98119c1a72b59cf59e683a64ce75852005ade6f7bff4af08e31f
+size 4657231202

model-00042-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3927ccbc2806d930d2af41ae82f31ed8c79ebe66a205001c51b09a7fcc2ae29
+size 4657231228

model-00044-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adbbb54abd36592f60db9460d84693304456fd84306f46fb75504b497bd3a037
+size 4657231254

model-00047-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08553574d456494e6dee01cf875e966c530b369ec07be20c05d4bd60b3c7ef8a
+size 4657231186

model-00048-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3e72c59af802312500f06daeb9050ef81171569cee14304fb8979eabfaf045b
+size 4657231180

model-00050-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9333ad2fc3b0e5840cb27037a45505353feb077e184ee6770af343c8cf9f444
+size 4657231190

model-00053-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dade8595e4f85dddbe5dc3e7bc95e83baeb41939d3bb0eaf086350a9b45bd7d
+size 4657231184

model-00056-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b2e460e389a66c54f2286feac2a059e7e600ac57b5a9175103523b35b9aec4a
+size 4657231214

model-00057-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96041398b4703db8a8cfd0e9ccdaa3638fe34e633efbed88bb2caa46885b0c58
+size 4657231244

model-00059-of-00059.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85e544b3b4ee82cc4f91b08a18439b262a2209dd9470098c4f223de75e87ab4f
+size 1774818666

preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "MiniMaxM3VLImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 451584,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "rescale_factor": 0.00392156862745098,
+  "size": [
+    672,
+    672
+  ],
+  "temporal_patch_size": 2
+}

processing_minimax.py

@@ -0,0 +1,254 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+"""
+MiniMax VL family HuggingFace-compatible Processor, ImageProcessor, VideoProcessor.
+"""
+
+import math
+import re
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torchvision
+from torchvision.transforms import InterpolationMode
+from transformers import BatchFeature
+from transformers.image_processing_utils_fast import (
+    BaseImageProcessorFast,
+    group_images_by_shape,
+    reorder_images,
+)
+from transformers.image_utils import PILImageResampling, SizeDict
+from transformers.processing_utils import (
+    ImagesKwargs,
+    ProcessingKwargs,
+    ProcessorMixin,
+    Unpack,
+    VideosKwargs,
+)
+from transformers.utils import TensorType
+from transformers.video_processing_utils import BaseVideoProcessor
+from transformers.video_utils import group_videos_by_shape, reorder_videos
+
+
+class MiniMaxVLProcessorKwargs(ProcessingKwargs, total=False):
+    _defaults = {
+        "videos_kwargs": {
+            "do_resize": False,
+            "return_metadata": True,
+        },
+    }
+
+
+class MiniMaxVLProcessor(ProcessorMixin):
+    IMAGE_TOKEN = "]<]image[>["
+    VIDEO_TOKEN = "]<]video[>["
+    VISION_START_TOKEN = "]<]start of image[>["
+    VISION_END_TOKEN = "]<]end of image[>["
+
+    def __init__(
+        self, image_processor=None, tokenizer=None, video_processor=None, **kwargs
+    ):
+        self.image_token_id = tokenizer.convert_tokens_to_ids(self.IMAGE_TOKEN)
+        self.video_token_id = tokenizer.convert_tokens_to_ids(self.VIDEO_TOKEN)
+        super().__init__(image_processor, tokenizer, video_processor)
+        # Video expansion also uses image start/end tokens. Separate video
+        # start/end tokens exist in the tokenizer, but the original MiniMax
+        # serving path did not use them; keep that behavior for compatibility.
+        self.vision_start_token_id = tokenizer.convert_tokens_to_ids(
+            self.VISION_START_TOKEN
+        )
+        self.vision_end_token_id = tokenizer.convert_tokens_to_ids(
+            self.VISION_END_TOKEN
+        )
+
+    def _prune_video_tokens(
+        self,
+        input_text: str,
+        video_segments: List[int],
+        video_token: str,
+    ) -> str:
+        """
+        Prune video tokens by temporal_patch_size (e.g., 2:1).
+
+        Expects the prompt to carry exactly sum(video_segments) video
+        tokens — i.e. one token per *sampled* frame. Then drops token.
+
+        Args:
+            input_text: prompt with N video_tokens per segment
+            video_segments: actual sampled frame count per video segment
+            video_token: the video token string, e.g. ']<]video[>['
+
+        Returns:
+            Pruned input_text with ~N/temporal_patch_size tokens per segment.
+        """
+        # If no videos or temporal_patch_size <= 1, no pruning needed
+        if not video_segments or self.video_processor.temporal_patch_size <= 1:
+            return input_text
+
+        # Split while keeping delimiters
+        special_tokens = [video_token]  # , image_token]
+        pattern = "|".join(map(re.escape, special_tokens))
+        parts = re.split(f"({pattern})", input_text)
+
+        def is_timestamp(text: str) -> bool:
+            """Check if text ends with timestamp format like ']<]0.0 seconds[>['"""
+            return (
+                text.endswith("seconds[>[")
+                or text.endswith("seconds[>[ ")
+                or text.endswith("seconds [>[")
+                or text.endswith("seconds [>[ ")
+            )
+
+        def extract_timestamp(text: str) -> str:
+            """Extract timestamp text from the end, starting from ']<]'"""
+            start_index = text.rfind("]<]")
+            if start_index == -1:
+                raise ValueError(f"Failed to extract timestamp: {text}")
+            return text[start_index:]
+
+        # Build new text with pruned video tokens
+        final_parts = []
+        current_seg_idx = 0  # Which video segment we're in
+        frame_in_seg = 0  # Frame index within current segment
+        last_timestamp_len = 0  # Length of timestamp to potentially remove
+
+        for part in parts:
+            if part == video_token:
+                if current_seg_idx < len(video_segments):
+                    if frame_in_seg % self.video_processor.temporal_patch_size == 0:
+                        # Keep this video token
+                        final_parts.append(part)
+                        frame_in_seg += 1
+                        if frame_in_seg >= video_segments[current_seg_idx]:
+                            current_seg_idx += 1
+                            frame_in_seg = 0
+                        last_timestamp_len = 0
+                    else:
+                        # Skip this video token
+                        frame_in_seg += 1
+                        if frame_in_seg >= video_segments[current_seg_idx]:
+                            current_seg_idx += 1
+                            frame_in_seg = 0
+                        # Remove the timestamp that was already appended
+                        if last_timestamp_len > 0:
+                            # Truncate the last part to remove timestamp
+                            assert len(final_parts) > 0
+                            final_parts[-1] = final_parts[-1][:-last_timestamp_len]
+                            last_timestamp_len = 0
+                else:
+                    # No more video segments, keep as is
+                    final_parts.append(part)
+                    last_timestamp_len = 0
+            else:
+                # Text part
+                final_parts.append(part)
+                # Check if this text ends with a timestamp
+                if is_timestamp(part):
+                    last_timestamp_len = len(extract_timestamp(part))
+                else:
+                    last_timestamp_len = 0
+
+        return "".join(final_parts)
+
+    def __call__(
+        self,
+        images=None,
+        text=None,
+        videos=None,
+        **kwargs: Unpack[MiniMaxVLProcessorKwargs],
+    ) -> BatchFeature:
+        output_kwargs = self._merge_kwargs(
+            MiniMaxVLProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if images is not None:
+            images_kwargs = output_kwargs["images_kwargs"]
+            image_inputs = self.image_processor(images=images, **images_kwargs)
+            image_grid_thw = image_inputs["image_grid_thw"]
+
+        else:
+            image_inputs = {}
+            image_grid_thw = None
+
+        if videos is not None:
+            videos_kwargs = output_kwargs["videos_kwargs"]
+            video_inputs = self.video_processor(videos=videos, **videos_kwargs)
+            video_grid_thw = video_inputs["video_grid_thw"]
+            if not kwargs.get("return_metadata"):
+                video_metadata = video_inputs.pop("video_metadata")
+            else:
+                video_metadata = video_inputs["video_metadata"]
+        else:
+            video_inputs = {}
+            video_grid_thw = None
+
+        if not isinstance(text, list):
+            text = [text]
+        text = text.copy()
+
+        # Expand image tokens
+        if image_grid_thw is not None:
+            merge_length = self.image_processor.merge_size**2
+            placeholder = "]<]placeholder[>["
+            index = 0
+            for i in range(len(text)):
+                while self.IMAGE_TOKEN in text[i]:
+                    num_tokens = image_grid_thw[index].prod() // merge_length
+                    text[i] = text[i].replace(
+                        self.IMAGE_TOKEN,
+                        self.VISION_START_TOKEN
+                        + placeholder * num_tokens
+                        + self.VISION_END_TOKEN,
+                        1,
+                    )
+                    index += 1
+                text[i] = text[i].replace(placeholder, self.IMAGE_TOKEN)
+
+        # Expand video tokens
+        if video_grid_thw is not None:
+            merge_length = self.image_processor.merge_size**2
+            placeholder = "]<]placeholder[>["
+            index = 0
+            for i in range(len(text)):
+                while self.VIDEO_TOKEN in text[i]:
+                    metadata = video_metadata[index]
+                    grid_t = video_grid_thw[index][0]
+                    frame_seqlen = video_grid_thw[index][1:].prod() // merge_length
+
+                    video_placeholder = ""
+                    for frame_idx in range(grid_t):
+                        if (
+                            metadata.fps is not None
+                            and metadata.frames_indices is not None
+                        ):
+                            ts = (
+                                metadata.frames_indices[
+                                    min(
+                                        frame_idx
+                                        * self.video_processor.temporal_patch_size,
+                                        len(metadata.frames_indices) - 1,
+                                    )
+                                ]
+                                / metadata.fps
+                            )
+                            video_placeholder += f"]<]{ts:.1f} seconds[>["
+                        video_placeholder += (
+                            self.VISION_START_TOKEN
+                            + placeholder * frame_seqlen
+                            + self.VISION_END_TOKEN
+                        )
+
+                    text[i] = text[i].replace(self.VIDEO_TOKEN, video_placeholder, 1)
+                    index += 1
+                text[i] = text[i].replace(placeholder, self.VIDEO_TOKEN)
+
+        # Tokenize
+        return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None)
+        text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
+
+        return BatchFeature(
+            data={**text_inputs, **image_inputs, **video_inputs},
+            tensor_type=return_tensors,
+        )

processor_config.json

@@ -0,0 +1,63 @@
+{
+  "chat_template": "{# ---------- special token variables ---------- #}\n{%- set ns_token               = ']<]minimax[>['                  -%}\n{%- set bod_token              = ']~!b['                          -%}\n{%- set bos_token              = ']~b]'                           -%}\n{%- set eos_token              = '[e~['                           -%}\n{%- set toolcall_begin_token   = ns_token ~ '<tool_call>'         -%}\n{%- set toolcall_end_token     = ns_token ~ '</tool_call>'        -%}\n{%- set think_begin_token      = '<mm:think>'                     -%}\n{%- set think_end_token        = '</mm:think>'                    -%}\n{%- set image_token            = ']<]image[>['                    -%}\n{%- set video_token            = ']<]video[>['                    -%}\n{#- Thinking mode: \"enabled\" / \"disabled\" / \"adaptive\" / not defined -#}\n{#- Recursive XML renderer for tool_call arguments ======================== -#}\n{#- None values are intentionally skipped in mapping iteration so that\n    `<key>null</key>` (which would round-trip to the literal string \"null\")\n    never appears in the rendered tool_call. The convention is: omit the\n    field entirely. The top-level `_args` loop applies the same rule.\n    The `val is none` branch below is a safety net only \u2014 upstream cleaning\n    (drop_none_in_tool_arguments) should ensure no None ever reaches here. -#}\n{%- macro to_xml(val, ns) -%}\n{%- if val is mapping -%}\n{%- for k, v in val.items() if v is not none -%}\n{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>\n{%- endfor -%}\n{%- elif val is iterable and val is not string -%}\n{%- for item in val -%}\n{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>\n{%- endfor -%}\n{%- elif val is none -%}\n{#- Should be unreachable when upstream cleaning is applied. -#}\n{%- elif val is boolean -%}\n{{ val | tojson }}\n{%- else -%}\n{{ val }}\n{%- endif -%}\n{%- endmacro -%}\n{#- Tool Rendering Functions ============================================== -#}\n{%- macro render_tool_namespace(namespace_name, tool_list) -%}\n{%- for tool in tool_list -%}\n<tool>{{ tool.function | tojson(ensure_ascii=False) }}</tool>\n{% endfor -%}\n{%- endmacro -%}\n{%- macro visible_text(content) -%}\n    {%- if content is string -%}\n        {{ content }}\n    {%- elif content is iterable and content is not mapping -%}\n        {%- for item in content -%}\n            {%- if item is mapping and item.type == 'text' -%}\n                {{- item.text }}\n            {%- elif item is mapping and item.type == 'image' -%}\n                {{- image_token }}\n            {%- elif item is mapping and item.type == 'video' -%}\n                {{- video_token}}\n            {%- elif item is string -%}\n                {{- item }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- elif content is none -%}\n        {{- '' }}\n    {%- else -%}\n        {{- content }}\n    {%- endif -%}\n{%- endmacro -%}\n{#- System Message Construction ============================================ -#}\n{%- macro build_system_message(system_message) -%}\n    {%- if system_message and system_message.content -%}\n        {{- visible_text(system_message.content) }}\n    {%- else -%}\n        {{- 'Your model version is MiniMax-M3, developed by MiniMax. Knowledge cutoff: January 2026. Founded in early 2022, MiniMax is a global AI foundation model company committed to advancing the frontiers of AI towards AGI.' }}\n    {%- endif -%}\n\n    {#- Thinking mode instructions -#}\n    {{- '\\n\\n<thinking_instructions>\\n' }}\n    {{- 'You have a thinking capability that allows you to reason step by step before responding. When thinking is enabled, wrap your reasoning in ' ~ think_begin_token ~ think_end_token ~ ' tags before your response. When thinking is disabled, begin your response directly after the ' ~ think_end_token ~ ' prefix. When thinking is adaptive, decide on your own whether to think for the current turn.\\n' }}\n    {%- if thinking_mode is defined -%}\n        {%- if thinking_mode == \"enabled\" -%}\n            {{- 'Current thinking mode: enabled. You MUST think step by step before every response, including after receiving function/tool results.\\n' }}\n        {%- elif thinking_mode == \"disabled\" -%}\n            {{- 'Current thinking mode: disabled. Do not output any thinking process.\\n' }}\n        {%- elif thinking_mode == \"adaptive\" -%}\n            {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\\n' }}\n        {%- endif -%}\n    {%- else -%}\n        {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\\n' }}\n    {%- endif -%}\n    {{- '</thinking_instructions>' }}\n{%- endmacro -%}\n{%- macro build_developer_message(developer_message) -%}\n    {%- if developer_message and developer_message.content -%}\n        {{- visible_text(developer_message.content) }}\n    {%- else -%}\n        {%- if model_identity is not defined -%}\n            {%- set model_identity = \"You are a helpful assistant.\" -%}\n        {%- endif -%}\n        {{- model_identity }}\n    {%- endif -%}\n{%- endmacro -%}\n{#- Main Template Logic ================================================= -#}\n{#- Role mapping: root -> system sp (high priority), system/developer -> developer sp (low priority) -#}\n{%- set system_message = none -%}\n{%- set developer_message = none -%}\n{%- set conversation_messages = messages -%}\n{%- if messages and messages[0].role == \"root\" -%}\n    {%- set system_message = messages[0] -%}\n    {%- set conversation_messages = messages[1:] -%}\n    {%- if conversation_messages and conversation_messages[0].role in [\"system\", \"developer\"] -%}\n        {%- set developer_message = conversation_messages[0] -%}\n        {%- set conversation_messages = conversation_messages[1:] -%}\n    {%- endif -%}\n{%- elif messages and messages[0].role in [\"system\", \"developer\"] -%}\n    {%- set developer_message = messages[0] -%}\n    {%- set conversation_messages = messages[1:] -%}\n{%- endif -%}\n{#- Render system sp (higher priority, root role only) -#}\n{{- bod_token ~ bos_token ~ 'system' ~ '\\n' }}\n{{- build_system_message(system_message) }}\n{{- eos_token ~ '\\n' }}\n\n{#- Render developer sp (lower priority: system/developer role + tools) -#}\n{{- bos_token ~ 'developer' ~ '\\n' }}\n{{- build_developer_message(developer_message) }}\n{%- if tools -%}\n    {{- '\\n\\n' ~ '# Tools' ~ '\\n' ~ 'You may call one or more tools to assist with the user query.\\nHere are the tools available in JSONSchema format:' ~ '\\n' }}\n    {{- '\\n' ~ '<tools>' ~ '\\n' }}\n    {{- render_tool_namespace(\"functions\", tools) }}\n    {{- '</tools>' ~ '\\n\\n' }}\n    {{- 'To call tools, wrap all invocations in a single ' ~ toolcall_begin_token ~ toolcall_end_token ~ ' block. Parameter values containing nested objects or arrays are recursively expanded into XML elements. Example:\\n' }}\n    {{- '\\n' ~ toolcall_begin_token ~ '\\n' }}\n    {{- ns_token + '<invoke name=\"tool-name-1\">' }}\n    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}\n    {{- ns_token + '<param-2>' }}\n    {{- ns_token + '<item>' }}\n    {{- ns_token + '<key-a>val-a' + ns_token + '</key-a>' }}\n    {{- ns_token + '<key-b>val-b' + ns_token + '</key-b>' }}\n    {{- ns_token + '</item>' }}\n    {{- ns_token + '</param-2>' }}\n    {{- ns_token + '</invoke>\\n' }}\n    {{- ns_token + '<invoke name=\"tool-name-2\">' }}\n    {{- ns_token + '<param-1>value-1' + ns_token + '</param-1>' }}\n    {{- ns_token + '</invoke>\\n' }}\n    {{- toolcall_end_token }}\n{%- endif -%}\n{{- eos_token ~ '\\n' }}\n\n{#- Render messages -#}\n{%- set last_tool_call = namespace(name=none) -%}\n{%- for message in conversation_messages -%}\n    {%- if message.role == 'assistant' -%}\n        {{- bos_token ~ 'ai' ~ '\\n' }}\n\n        {%- set reasoning_content = '' %}\n        {%- set content = visible_text(message.content) %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if think_end_token in content %}\n                {%- set reasoning_content = content.split(think_end_token)[0].strip('\\n').split(think_begin_token)[-1].strip('\\n') %}\n                {%- set content = content.split(think_end_token)[-1].strip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n\n        {%- if reasoning_content -%}\n            {#- Render thinking for every assistant turn (all-turn visible) -#}\n            {{- think_begin_token ~ reasoning_content ~ think_end_token }}\n        {%- else -%}\n            {#- No thinking rendered \u2192 prefix with think_end_token -#}\n            {{- think_end_token }}\n        {%- endif -%}\n\n        {%- if content -%}\n            {{- content }}\n        {%- endif -%}\n        {%- if message.tool_calls -%}\n            {{- toolcall_begin_token ~ '\\n' }}\n\n            {%- for tool_call in message.tool_calls -%}\n                {%- if tool_call.function -%}\n                    {%- set tool_call = tool_call.function -%}\n                {%- endif -%}\n{{- ns_token + '<invoke name=\"' + tool_call.name + '\">' }}\n{%- set _args = tool_call.arguments -%}\n{%- for k, v in _args.items() if v is not none %}\n{{- ns_token + '<' + k + '>' -}}\n{{- to_xml(v, ns_token) -}}\n{{- ns_token + '</' + k + '>' }}\n{%- endfor -%}\n{{- ns_token + '</invoke>' ~ '\\n' }}\n            {%- endfor -%}\n\n            {{- toolcall_end_token }}\n            {%- if message.tool_calls[-1].function -%}\n                {%- set last_tool_call.name = message.tool_calls[-1].function.name -%}\n            {%- else -%}\n                {%- set last_tool_call.name = message.tool_calls[-1].name -%}\n            {%- endif -%}\n        {%- else -%}\n            {%- set last_tool_call.name = none -%}\n        {%- endif -%}\n        {{- eos_token ~ '\\n' }}\n\n    {%- elif message.role == 'tool' -%}\n        {%- if last_tool_call.name is none -%}\n            {{- raise_exception(\"Message has tool role, but there was no previous assistant message with a tool call!\") }}\n        {%- endif -%}\n        {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%}\n            {{- bos_token ~ 'tool' }}\n        {%- endif -%}\n        {{- '\\n<response>' }}\n        {%- if message.content is string -%}\n            {{- message.content }}\n        {%- else -%}\n            {%- for tr in message.content -%}\n                {%- if tr is mapping and tr.type is defined and tr.type == 'image' -%}\n                    {{- image_token }}\n                {%- elif tr is mapping and tr.type is defined and tr.type == 'video' -%}\n                    {{- video_token }}\n                {%- else -%}\n                    {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }}\n                {%- endif -%}\n            {%- endfor -%}\n        {%- endif -%}\n        {{- '</response>' }}\n        {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%}\n            {{- eos_token ~ '\\n' -}}\n        {%- endif -%}\n\n    {%- elif message.role == 'user' -%}\n        {{- bos_token ~ 'user' ~ '\\n' }}\n        {{- visible_text(message.content) }}\n        {{- eos_token ~ '\\n' }}\n    {%- endif -%}\n{%- endfor -%}\n\n{#- Generation prompt -#}\n{%- if add_generation_prompt -%}\n{{- bos_token ~ 'ai' ~ '\\n' }}\n{%- if thinking_mode is defined and thinking_mode == \"disabled\" -%}\n    {{- think_end_token }}\n{%- elif thinking_mode is defined and thinking_mode == \"adaptive\" -%}\n    {#- adaptive: no prefix, let model decide -#}\n{%- elif thinking_mode is defined and thinking_mode == \"enabled\" -%}\n    {#- enabled or not defined: default to think -#}\n    {{- think_begin_token }}\n{%- else -%}\n    {#- adaptive: no prefix, let model decide -#}\n{%- endif -%}\n{%- endif -%}\n",
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_processor_type": "MiniMaxM3VLImageProcessor",
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_pixels": 451584,
+    "merge_size": 2,
+    "min_pixels": 3136,
+    "patch_size": 14,
+    "rescale_factor": 0.00392156862745098,
+    "size": [
+      672,
+      672
+    ],
+    "temporal_patch_size": 2
+  },
+  "processor_class": "MiniMaxM3VLProcessor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": false,
+    "fps": 1.0,
+    "image_mean": [
+      0.48145466,
+      0.4578275,
+      0.40821073
+    ],
+    "image_std": [
+      0.26862954,
+      0.26130258,
+      0.27577711
+    ],
+    "max_frames": 768,
+    "max_pixels": 602112,
+    "merge_size": 2,
+    "min_frames": 4,
+    "min_pixels": 3136,
+    "patch_size": 14,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "height": 672,
+      "width": 672
+    },
+    "temporal_patch_size": 2,
+    "total_pixels": 45158400,
+    "video_processor_type": "MiniMaxM3VLVideoProcessor"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "]~b]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "[e~[",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae73e31fefce04b005cb41c6781389426fae1a8553b6e58d29f133eaa31ebfb5
+size 15524484

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "]~b]",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "[e~[",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 40960000,
+  "pad_token": "]!p~[",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[e~["
+}

video_preprocessor_config.json

@@ -0,0 +1,32 @@
+{
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "do_sample_frames": false,
+  "fps": 1.0,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_frames": 768,
+  "max_pixels": 602112,
+  "merge_size": 2,
+  "min_frames": 4,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 672,
+    "width": 672
+  },
+  "temporal_patch_size": 2,
+  "total_pixels": 45158400,
+  "video_processor_type": "MiniMaxM3VLVideoProcessor"
+}

video_processor.py

@@ -0,0 +1,208 @@
+# Copyright 2023-2024 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+"""
+MiniMax VL family HuggingFace-compatible VideoProcessor.
+"""
+
+import math
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torchvision
+from torchvision.transforms import InterpolationMode
+from transformers import BatchFeature
+from transformers.image_utils import PILImageResampling, SizeDict
+from transformers.processing_utils import (
+    Unpack,
+    VideosKwargs,
+)
+from transformers.utils import TensorType
+from transformers.video_processing_utils import BaseVideoProcessor
+from transformers.video_utils import group_videos_by_shape, reorder_videos
+
+MAX_RATIO = 200
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(
+    height: int,
+    width: int,
+    factor: int = 28,
+    min_pixels: int = 4 * 28 * 28,
+    max_pixels: int = 451584,
+) -> tuple[int, int]:
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, "
+            f"got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+class MiniMaxM3VLVideoProcessorKwargs(VideosKwargs, total=False):
+    patch_size: int
+    temporal_patch_size: int
+    merge_size: int
+    min_pixels: int
+    max_pixels: int
+    total_pixels: int
+    min_frames: int
+    max_frames: int
+    fps: float | int
+
+
+class MiniMaxM3VLVideoProcessor(BaseVideoProcessor):
+    do_resize = True
+    resample = PILImageResampling.BICUBIC
+    size = {"height": 672, "width": 672}
+    default_to_square = False
+    do_rescale = True
+    rescale_factor = 1 / 255
+    do_normalize = True
+    image_mean = [0.48145466, 0.4578275, 0.40821073]
+    image_std = [0.26862954, 0.26130258, 0.27577711]
+    do_convert_rgb = True
+    do_sample_frames = False
+    patch_size = 14
+    temporal_patch_size = 2
+    merge_size = 2
+    min_pixels = 4 * 28 * 28
+    max_pixels = 768 * 28 * 28                  # 602,112
+    total_pixels = int(64000 * 28 * 28 * 0.9)   # ~45M, ~64k tokens budget
+    fps = 1.0
+    min_frames = 4
+    max_frames = 768
+    valid_kwargs = MiniMaxM3VLVideoProcessorKwargs
+    model_input_names = ["pixel_values_videos", "video_grid_thw"]
+
+    def __init__(self, **kwargs: Unpack[MiniMaxM3VLVideoProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    def _preprocess(
+        self,
+        videos: List[torch.Tensor],
+        do_convert_rgb: bool,
+        do_resize: bool,
+        size: SizeDict,
+        resample: PILImageResampling | InterpolationMode | int | None,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: float | List[float] | None,
+        image_std: float | List[float] | None,
+        patch_size: int,
+        temporal_patch_size: int,
+        merge_size: int,
+        min_pixels: int,
+        max_pixels: int,
+        return_tensors: str | TensorType | None = None,
+        **kwargs,
+    ) -> BatchFeature:
+        grouped_videos, grouped_videos_index = group_videos_by_shape(videos)
+        resized_videos_grouped = {}
+        factor = patch_size * merge_size
+        for shape, stacked_videos in grouped_videos.items():
+            batch_size, num_frames, channels, height, width = stacked_videos.shape
+            resized_height, resized_width = height, width
+            if do_resize:
+                resized_height, resized_width = smart_resize(
+                    height, width, factor=factor,
+                    min_pixels=min_pixels, max_pixels=max_pixels,
+                )
+                stacked_videos = stacked_videos.view(
+                    batch_size * num_frames, channels, height, width
+                )
+                stacked_videos = self.resize(
+                    stacked_videos,
+                    size=SizeDict(height=resized_height, width=resized_width),
+                    resample=resample,
+                )
+                stacked_videos = stacked_videos.view(
+                    batch_size,
+                    num_frames,
+                    channels,
+                    resized_height,
+                    resized_width,
+                )
+            resized_videos_grouped[shape] = stacked_videos
+        resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index)
+
+        grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos)
+        processed_videos_grouped = {}
+        processed_grids = {}
+        for shape, stacked_videos in grouped_videos.items():
+            resized_height, resized_width = stacked_videos.shape[-2:]
+            patches = self.rescale_and_normalize(
+                stacked_videos,
+                do_rescale,
+                rescale_factor,
+                do_normalize,
+                image_mean,
+                image_std,
+            )
+
+            if pad := -patches.shape[1] % temporal_patch_size:
+                repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1)
+                patches = torch.cat([patches, repeats], dim=1)
+
+            batch_size, grid_t, channels = patches.shape[:3]
+            grid_t = grid_t // temporal_patch_size
+            grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
+
+            patches = patches.view(
+                batch_size,
+                grid_t,
+                temporal_patch_size,
+                channels,
+                grid_h // merge_size,
+                merge_size,
+                patch_size,
+                grid_w // merge_size,
+                merge_size,
+                patch_size,
+            )
+            patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9)
+            flatten_patches = patches.reshape(
+                batch_size,
+                grid_t * grid_h * grid_w,
+                channels * temporal_patch_size * patch_size * patch_size,
+            )
+
+            processed_videos_grouped[shape] = flatten_patches
+            processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size
+
+        processed_videos = reorder_videos(
+            processed_videos_grouped, grouped_videos_index
+        )
+        processed_grids = reorder_videos(processed_grids, grouped_videos_index)
+        pixel_values_videos = torch.cat(processed_videos, dim=0)
+        video_grid_thw = torch.tensor(processed_grids, dtype=torch.long)
+
+        return BatchFeature(
+            data={
+                "pixel_values_videos": pixel_values_videos,
+                "video_grid_thw": video_grid_thw,
+            },
+            tensor_type=return_tensors,
+        )