Spaces:
Paused
Paused
File size: 4,867 Bytes
0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 df28248 0633a27 0a882d1 0633a27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | #!/bin/bash
set -e
echo "Starting Bonsai-demo entrypoint..."
# ── Hardcoded models (edit and push to change) ───────────────────────────────
# Format: "repo|file"
MODELS=(
"prism-ml/Bonsai-8B-gguf|Bonsai-8B-Q1_0.gguf"
"prism-ml/Ternary-Bonsai-8B-gguf|Ternary-Bonsai-8B-Q2_0.gguf"
)
MODEL_DIR="/app/models"
mkdir -p "$MODEL_DIR"
download_model() {
local repo file path retries=5
repo="$1"
file="$2"
path="$MODEL_DIR/$file"
local url="https://huggingface.co/$repo/resolve/main/$file"
for attempt in $(seq 1 $retries); do
echo "Downloading (attempt $attempt/$retries): $url"
rm -f "$path"
if curl -fL --retry 3 --retry-delay 5 -C - -o "$path" \
${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} \
"$url" 2>&1; then
echo "Downloaded: $(ls -lh "$path")"
return 0
fi
echo "Download failed, retrying in 10s..."
sleep 10
done
echo "ERROR: Failed to download $url after $retries attempts"
exit 1
}
for entry in "${MODELS[@]}"; do
download_model "${entry%%|*}" "${entry#*|}"
done
MODELS_MAX=${#MODELS[@]}
# ── Dashboard auth ────────────────────────────────────────────────────────────
if [ -n "$DASHBOARD_KEY" ]; then
HASH=$(openssl passwd -apr1 "$DASHBOARD_KEY")
echo "admin:$HASH" > /tmp/.htpasswd
echo "Dashboard auth: enabled (user=admin)"
else
echo "WARNING: DASHBOARD_KEY not set, /dash-2e215f981f3f is unprotected"
printf 'admin:$apr1$open$open\n' > /tmp/.htpasswd
fi
# ── nginx temp dirs ───────────────────────────────────────────────────────────
mkdir -p /tmp/nginx-{client-body,proxy,fastcgi,uwsgi,scgi}
# ── MCP servers (shared across all GPU backends) ─────────────────────────────
echo ""
echo "=== Starting MCP servers ==="
# Brave Search MCP (native HTTP) on 127.0.0.1:8001/mcp
if [ -n "$BRAVE_API_KEY" ]; then
echo "Starting Brave Search MCP on 127.0.0.1:8001"
brave-search-mcp-server --transport http --host 127.0.0.1 --port 8001 &
else
echo "BRAVE_API_KEY not set -- skipping Brave Search MCP"
fi
# ArXiv MCP (stdio) bridged via supergateway on 127.0.0.1:8002/sse
echo "Starting ArXiv MCP on 127.0.0.1:8002 (via supergateway)"
supergateway --stdio "arxiv-mcp-server" --port 8002 --baseUrl http://127.0.0.1:8002 &
sleep 1
# ── Detect GPUs and start one llama-server per GPU ───────────────────────────
GPU_COUNT=$(nvidia-smi -L 2>/dev/null | wc -l || echo 1)
echo "GPUs detected: $GPU_COUNT"
BACKENDS=""
for i in $(seq 0 $((GPU_COUNT - 1))); do
PORT=$((7861 + i))
echo "Starting llama-server on GPU $i → port $PORT"
CUDA_VISIBLE_DEVICES=$i /app/bin/llama-server \
--models-dir "$MODEL_DIR" \
--models-max "$MODELS_MAX" \
--host 127.0.0.1 \
--port "$PORT" \
-ngl 99 \
-fa on \
-np 4 \
-c 262144 \
--metrics \
--jinja \
--webui-mcp-proxy \
--temp 0.5 --top-p 0.85 --top-k 20 --min-p 0 \
--reasoning-budget 0 --reasoning-format none \
--chat-template-kwargs '{"enable_thinking": false}' \
--log-disable &
BACKENDS="$BACKENDS server 127.0.0.1:$PORT;\n"
done
printf "upstream llama_backends {\n least_conn;\n${BACKENDS}}\n" > /tmp/nginx-upstream.conf
# ── Write stub files so /gpu and /analytics never 404 before first tick ──────
echo '{"ts":null,"gpus":[]}' > /tmp/gpu-stats.json
echo '# waiting for first metrics scrape...' > /tmp/llama-metrics.txt
echo '{"updated_at":null,"summary_24h":{"requests":0,"unique_users":0},"summary_7d":{"requests":0,"unique_users":0},"summary_total":{"requests":0,"unique_users":0},"requests_by_hour":[],"requests_by_day":[],"top_users":[]}' > /tmp/analytics.json
# ── Start metrics pusher with watchdog ────────────────────────────────────────
start_metrics_pusher() {
while true; do
echo "[watchdog] Starting metrics_pusher.py..."
python3 /app/metrics_pusher.py || true
echo "[watchdog] metrics_pusher.py exited — restarting in 5s..."
sleep 5
done
}
start_metrics_pusher &
echo ""
echo "=== Bonsai-demo ==="
echo " Models: $(ls "$MODEL_DIR"/*.gguf | xargs -n1 basename | tr '\n' ', ' | sed 's/,$//')"
echo " GPUs: $GPU_COUNT"
echo " Port: 7860 (nginx → llama-server)"
echo ""
exec nginx -c /app/nginx.conf
|