add ternary + mcp

.gitattributes

@@ -34,4 +34,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 bin/*.so.* filter=lfs diff=lfs merge=lfs -text
+bin/libggml-*.so filter=lfs diff=lfs merge=lfs -text
 bin/llama-* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,14 +1,27 @@
 FROM nvidia/cuda:12.8.0-runtime-ubuntu22.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-        curl ca-certificates git libgomp1 nginx openssl python3 \
+        curl ca-certificates git libgomp1 libssl3 nginx openssl python3 \
     && rm -rf /var/lib/apt/lists/* \
     && chown -R 1000:1000 /var/lib/nginx /var/log/nginx /run
 
+# Node.js 20 (for JS-based MCP servers + supergateway bridge)
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
+    && apt-get install -y --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/*
+
 RUN useradd -m -u 1000 user
 USER user
 ENV HOME=/home/user PATH="/home/user/.local/bin:/app/bin:$PATH"
 
+# uv + arxiv-mcp-server (Python, stdio — bridged to HTTP via supergateway)
+RUN curl -fsSL https://astral.sh/uv/install.sh | sh \
+    && /home/user/.local/bin/uv tool install 'arxiv-mcp-server[pdf]'
+
+# Brave Search MCP (native HTTP) + supergateway (stdio→SSE bridge)
+RUN npm config set prefix /home/user/.local \
+    && npm install -g @brave/brave-search-mcp-server supergateway
+
 WORKDIR /app
 
 COPY --chown=user bin/ /app/bin/

README.md

@@ -1,5 +1,5 @@
 ---
-title: Bonsai 1-bit GPU
+title: Bonsai {1, 1.58}-bit GPU
 emoji: 🌿
 colorFrom: green
 colorTo: blue
@@ -7,11 +7,14 @@ sdk: docker
 app_port: 7860
 suggested_hardware: l40sx1
 pinned: true
-short_description: Run 1-bit Bonsai LLMs on GPUs
+short_description: Run {1, 1.58}-bit Bonsai LLMs on GPUs
 models:
   - prism-ml/Bonsai-8B-gguf
   - prism-ml/Bonsai-4B-gguf
   - prism-ml/Bonsai-1.7B-gguf
+  - prism-ml/Ternary-Bonsai-8B-gguf
+  - prism-ml/Ternary-Bonsai-4B-gguf
+  - prism-ml/Ternary-Bonsai-1.7B-gguf
 ---
 
 # Bonsai Demo
@@ -20,20 +23,36 @@ Interactive demo for **[Bonsai](https://huggingface.co/collections/prism-ml/bons
 
 Bonsai models run at **true 1-bit precision** — every weight is a single bit. An 8B model fits in **1.15 GB**, a 1.7B model in just **240 MB**. Small enough to run in a browser, on a phone, or on any laptop — while remaining competitive with full-precision models on benchmarks.
 
-> **This demo will be available for a limited time (approximately 1–2 weeks).** Enjoy it while it lasts!
+**Ternary-Bonsai** is the 1.58-bit sibling series — each weight is one of {−1, 0, +1}. It trades a bit of size for a quality bump over the pure 1-bit models. An 8B Ternary-Bonsai fits in **2.03 GB**, a 1.7B in just **430 MB**.
+
+> **This demo will be available for a limited time.** Enjoy it while it lasts!
 
 ## Highlights
 
 Bonsai-8B fits in **1.15 GB** (14x smaller than FP16) and generates at **~330 tok/s** on an L40S (6.3x faster than FP16). Scores 70.5 average across 6 benchmark tasks, competitive with full-precision 8B models.
 
+**Ternary-Bonsai-8B** (1.58-bit, 2.03 GB) is also available in the chat UI — pick it from the model dropdown for a quality bump at a modest size increase over the 1-bit models.
+
+**Tool calling & MCP**: the chat UI supports OpenAI-style tool calling and comes with Brave Search and ArXiv MCP servers pre-wired — open the MCP tab in the chat settings to use them.
+
 ## Models
 
+### 1-bit Bonsai
+
 | Model | Size | GGUF | MLX |
 |---|---|---|---|
 | **Bonsai-8B** | 1.15 GB | [prism-ml/Bonsai-8B-gguf](https://huggingface.co/prism-ml/Bonsai-8B-gguf) | [prism-ml/Bonsai-8B-mlx-1bit](https://huggingface.co/prism-ml/Bonsai-8B-mlx-1bit) |
 | **Bonsai-4B** | 570 MB | [prism-ml/Bonsai-4B-gguf](https://huggingface.co/prism-ml/Bonsai-4B-gguf) | [prism-ml/Bonsai-4B-mlx-1bit](https://huggingface.co/prism-ml/Bonsai-4B-mlx-1bit) |
 | **Bonsai-1.7B** | 240 MB | [prism-ml/Bonsai-1.7B-gguf](https://huggingface.co/prism-ml/Bonsai-1.7B-gguf) | [prism-ml/Bonsai-1.7B-mlx-1bit](https://huggingface.co/prism-ml/Bonsai-1.7B-mlx-1bit) |
 
+### 1.58-bit Ternary-Bonsai
+
+| Model | Size | GGUF |
+|---|---|---|
+| **Ternary-Bonsai-8B** | 2.03 GB | [prism-ml/Ternary-Bonsai-8B-gguf](https://huggingface.co/prism-ml/Ternary-Bonsai-8B-gguf) |
+| **Ternary-Bonsai-4B** | 1.00 GB | [prism-ml/Ternary-Bonsai-4B-gguf](https://huggingface.co/prism-ml/Ternary-Bonsai-4B-gguf) |
+| **Ternary-Bonsai-1.7B** | 430 MB | [prism-ml/Ternary-Bonsai-1.7B-gguf](https://huggingface.co/prism-ml/Ternary-Bonsai-1.7B-gguf) |
+
 ## Resources
 
 - [1-bit Bonsai Whitepaper](https://github.com/PrismML-Eng/Bonsai-demo/blob/main/1-bit-bonsai-8b-whitepaper.pdf)

bin/libggml-base.so.0

@@ -1 +1 @@
-libggml-base.so.0.9.7
+libggml-base.so.0.9.11

bin/{libggml-base.so.0.9.7 → libggml-base.so.0.9.11}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34c567f790d8f6b6ce63ae446431d737e382d10bbb7f634e79855749d75c79a6
-size 787520
+oid sha256:519961d682cb8b740fec46229a918ecae9b7dd072d7d0c40cc5ce0de4c3c0f85
+size 845400

bin/{libllama.so.0.0.8190 → libggml-cpu-alderlake.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2303b49c572d5b7b36123046cad7469d8b3caf8778834b37dc6df546fb341588
-size 3203944
+oid sha256:09c2d6908b8bab3e2503d9d4991be2465ddc86e4bd7a3c5bb1f2dcac77d7974e
+size 1116024

bin/{libggml-cpu.so.0.9.7 → libggml-cpu-cannonlake.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57b6c7bf5e9424bc52df6fc8f5503b22f029df912fee65f342b441d5032fb8e3
-size 1107144
+oid sha256:5ca4934d1a66c3a9e28a79eade62a5ca6310060ea5503965de1e9c6b6bf2dc31
+size 1261008

bin/libggml-cpu-cascadelake.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc5f7d431f0ef0cb1d93a3ae217a5980029a8c4855bbac0b73383a89b8ea3b3e
+size 1261008

bin/libggml-cpu-cooperlake.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb9f78486f6febcc349290a537d33812ed354684cb2a40003138c8104ba3a212
+size 1256976

bin/libggml-cpu-haswell.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af894b9c5dd734af5e8268eec8410ff0e0d6f672180a1b4f4f86e9fac23b928c
+size 1120120

bin/libggml-cpu-icelake.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79a268a1247da905951bfecf391b81798dd5dba8a08a25e44f663adee02eceb6
+size 1261008

bin/libggml-cpu-ivybridge.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eef94fd02e558f9fcdb438cb4298b3a42b96ae7f50ee0a9f4dc3649a19fa821
+size 1069488

bin/libggml-cpu-piledriver.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4541790316c2273f736b96de564aa90862e7edb6be78a704a3108ceb9e88f7da
+size 1065392

bin/libggml-cpu-sandybridge.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee54b58402ef7959d8a2b93545be6fd4cc155367b682a7d4feec314a7d73b468
+size 1060176

bin/libggml-cpu-sapphirerapids.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:210624b50dd11f91aef30477e12c5ecdb27da514063a413464c205df1a06b6b0
+size 1527440

bin/libggml-cpu-skylakex.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f844ef94bad71edae865c147d9de55d0e52f8ee75f5cfc518a238a2e976092c4
+size 1261008

bin/{libmtmd.so.0.0.8190 → libggml-cpu-sse42.so}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff0f675a01f13961e6f545ef414ae6cc50fcfa65379aa9ca1032437cd16194a3
-size 943272
+oid sha256:7ce33ec9c53629a14c2ee94ce515913a53cb642d36ac15bc241700e1a48c36a9
+size 861016

bin/libggml-cpu-x64.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d051bc7971d8febf2cb1307b6259018ca83870c72a1e98c36ce328ded3e70695
+size 852960

bin/libggml-cpu-zen4.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6377df73d036b3e3200087f16277dbd51b07eb6e043c7d239364c730bfff82e
+size 1256976

bin/libggml-cpu.so

@@ -1 +0,0 @@
-libggml-cpu.so.0

bin/libggml-cpu.so.0

@@ -1 +0,0 @@
-libggml-cpu.so.0.9.7

bin/libggml-cuda.so

@@ -1 +0,0 @@
-libggml-cuda.so.0

bin/libggml-cuda.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26bde851064e0e6cf25a51a2d5f6b2121e9f173112366c648db30c6e83011a85
+size 62558664

bin/libggml-cuda.so.0

@@ -1 +0,0 @@
-libggml-cuda.so.0.9.7

bin/libggml-cuda.so.0.9.7

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f9274ff637e09a5beb5d2ec55a475790bbd22ea3b6095b975ab4bbc18001ccd9
-size 55252864

bin/libggml.so.0

@@ -1 +1 @@
-libggml.so.0.9.7
+libggml.so.0.9.11

bin/{libggml.so.0.9.7 → libggml.so.0.9.11}

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cebe9400f91bbea8c0c1663d8a7582299835bd1d040dcd4a326ac067b4adddd9
-size 55184
+oid sha256:b5f77f5eb813abd62f00a19a77acbb02d8257838bb31298e566029c26d53686a
+size 54936

bin/libllama.so.0

@@ -1 +1 @@
-libllama.so.0.0.8190
+libllama.so.0.0.8799

bin/libllama.so.0.0.8799

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b46095aee60fa50ff13916c4f2c5370404d10c779409d11ec3a1a6e2c0e6781
+size 3274176

bin/libmtmd.so.0

@@ -1 +1 @@
-libmtmd.so.0.0.8190
+libmtmd.so.0.0.8799

bin/libmtmd.so.0.0.8799

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b4a300955c9404de480b2ae78e51da5d6fa365417cd147861c3f45365bd1143
+size 1117184

bin/llama-bench

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52825c5cbf617cd86db04a143188bee3d666e789168a897ced04001882afe2aa
-size 592680
+oid sha256:de3f39974acab8b31b45a87f5e487ba7c40ae6d4613abf06c2cdc8e17686515e
+size 2158088

bin/llama-cli

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4c09f893698bf498dc872b950635bcae961de90ce67dadc83c62d34c9812ead0
-size 5665432
+oid sha256:525b05afaccb7616203e1fb8c325375ce44d0b5cc6ab4a643b03123f920ef5cd
+size 6047952

bin/llama-completion

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b126040b118cb1bf0c041e2951e78a00aa2c619c414cc380b361c59c1d5980d
-size 4642520
+oid sha256:11fe8f49f3459dc33fade5018962dd655727b34aa94fb10d5c09d9ab4d99a764
+size 4979792

bin/llama-perplexity

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:445b36510eaa53b9ee1ee641a5672b0dd7642be921a2a043f7ebd639b084dd64
-size 4680640
+oid sha256:57d5d0fa5b6dca19860214d14ea3657dc0790af631416951f4f082c062ca5f53
+size 5021280

bin/llama-quantize

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6114cda3bf733678f49dab714d689b53f8505889ccae88650003d7a86a90f783
-size 442464
+oid sha256:55a7a8c1669f412ca96987944c2def5b1c6820e5d5c6a2614128f4b5e340b0b6
+size 457888

bin/llama-server

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b33f7d0abc362fdd49175df2b66741b4784c737f348cba5175220c6c871246dd
-size 7354688
+oid sha256:15c431ef81e57245b64bde067025f4732f7f61e6d47761329fb00881b9fd3bb7
+size 13416216

entrypoint.sh

@@ -3,30 +3,28 @@ set -e
 
 echo "Starting Bonsai-demo entrypoint..."
 
-# ── Validate required secrets ─────────────────────────────────────────────────
-if [ -z "$MODEL_REPO" ] || [ -z "$MODEL_FILE" ]; then
-    echo ""
-    echo "ERROR: MODEL_REPO and MODEL_FILE secrets must be set."
-    echo "  Go to Space Settings → Repository Secrets and add:"
-    echo "    MODEL_REPO = prism-ml/Bonsai-8B-gguf"
-    echo "    MODEL_FILE = Bonsai-8B.gguf"
-    echo ""
-    exit 1
-fi
+# ── Hardcoded models (edit and push to change) ───────────────────────────────
+# Format: "repo|file"
+MODELS=(
+    "prism-ml/Bonsai-8B-gguf|Bonsai-8B-Q1_0.gguf"
+    "prism-ml/Ternary-Bonsai-8B-gguf|Ternary-Bonsai-8B-Q2_0.gguf"
+)
 
 MODEL_DIR="/app/models"
 mkdir -p "$MODEL_DIR"
 
 download_model() {
     local repo file path retries=5
-    repo=$(echo "$1" | tr -d '[:space:]')
-    file=$(echo "$2" | tr -d '[:space:]')
+    repo="$1"
+    file="$2"
     path="$MODEL_DIR/$file"
     local url="https://huggingface.co/$repo/resolve/main/$file"
     for attempt in $(seq 1 $retries); do
         echo "Downloading (attempt $attempt/$retries): $url"
         rm -f "$path"
-        if curl -fL --retry 3 --retry-delay 5 -C - -o "$path" "$url" 2>&1; then
+        if curl -fL --retry 3 --retry-delay 5 -C - -o "$path" \
+            ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} \
+            "$url" 2>&1; then
             echo "Downloaded: $(ls -lh "$path")"
             return 0
         fi
@@ -37,23 +35,10 @@ download_model() {
     exit 1
 }
 
-# Primary model (required)
-MODEL_REPO=$(echo "$MODEL_REPO" | tr -d '[:space:]')
-MODEL_FILE=$(echo "$MODEL_FILE" | tr -d '[:space:]')
-download_model "$MODEL_REPO" "$MODEL_FILE"
-
-# Additional models (optional)
-MODELS_MAX=1
-for suffix in B C; do
-    repo_var="MODEL_REPO_${suffix}"
-    file_var="MODEL_FILE_${suffix}"
-    repo=$(echo "${!repo_var:-}" | tr -d '[:space:]')
-    file=$(echo "${!file_var:-}" | tr -d '[:space:]')
-    if [ -n "$repo" ] && [ -n "$file" ]; then
-        download_model "$repo" "$file"
-        MODELS_MAX=$((MODELS_MAX + 1))
-    fi
+for entry in "${MODELS[@]}"; do
+    download_model "${entry%%|*}" "${entry#*|}"
 done
+MODELS_MAX=${#MODELS[@]}
 
 # ── Dashboard auth ────────────────────────────────────────────────────────────
 if [ -n "$DASHBOARD_KEY" ]; then
@@ -68,6 +53,24 @@ fi
 # ── nginx temp dirs ───────────────────────────────────────────────────────────
 mkdir -p /tmp/nginx-{client-body,proxy,fastcgi,uwsgi,scgi}
 
+# ── MCP servers (shared across all GPU backends) ─────────────────────────────
+echo ""
+echo "=== Starting MCP servers ==="
+
+# Brave Search MCP (native HTTP) on 127.0.0.1:8001/mcp
+if [ -n "$BRAVE_API_KEY" ]; then
+    echo "Starting Brave Search MCP on 127.0.0.1:8001"
+    brave-search-mcp-server --transport http --host 127.0.0.1 --port 8001 &
+else
+    echo "BRAVE_API_KEY not set -- skipping Brave Search MCP"
+fi
+
+# ArXiv MCP (stdio) bridged via supergateway on 127.0.0.1:8002/sse
+echo "Starting ArXiv MCP on 127.0.0.1:8002 (via supergateway)"
+supergateway --stdio "arxiv-mcp-server" --port 8002 --baseUrl http://127.0.0.1:8002 &
+
+sleep 1
+
 # ── Detect GPUs and start one llama-server per GPU ───────────────────────────
 GPU_COUNT=$(nvidia-smi -L 2>/dev/null | wc -l || echo 1)
 echo "GPUs detected: $GPU_COUNT"
@@ -77,7 +80,8 @@ for i in $(seq 0 $((GPU_COUNT - 1))); do
     PORT=$((7861 + i))
     echo "Starting llama-server on GPU $i → port $PORT"
     CUDA_VISIBLE_DEVICES=$i /app/bin/llama-server \
-        -m "$MODEL_DIR/$MODEL_FILE" \
+        --models-dir "$MODEL_DIR" \
+        --models-max "$MODELS_MAX" \
         --host 127.0.0.1 \
         --port "$PORT" \
         -ngl 99 \
@@ -85,6 +89,8 @@ for i in $(seq 0 $((GPU_COUNT - 1))); do
         -np 4 \
         -c 262144 \
         --metrics \
+        --jinja \
+        --webui-mcp-proxy \
         --temp 0.5 --top-p 0.85 --top-k 20 --min-p 0 \
         --reasoning-budget 0 --reasoning-format none \
         --chat-template-kwargs '{"enable_thinking": false}' \