entrypoint.sh

#!/bin/bash
set -e

echo "Starting Bonsai-demo entrypoint..."

# ── Hardcoded models (edit and push to change) ───────────────────────────────
# Format: "repo|file"
MODELS=(
    "prism-ml/Bonsai-8B-gguf|Bonsai-8B-Q1_0.gguf"
    "prism-ml/Ternary-Bonsai-8B-gguf|Ternary-Bonsai-8B-Q2_0.gguf"
)

MODEL_DIR="/app/models"
mkdir -p "$MODEL_DIR"

download_model() {
    local repo file path retries=5
    repo="$1"
    file="$2"
    path="$MODEL_DIR/$file"
    local url="https://huggingface.co/$repo/resolve/main/$file"
    for attempt in $(seq 1 $retries); do
        echo "Downloading (attempt $attempt/$retries): $url"
        rm -f "$path"
        if curl -fL --retry 3 --retry-delay 5 -C - -o "$path" \
            ${HF_TOKEN:+-H "Authorization: Bearer $HF_TOKEN"} \
            "$url" 2>&1; then
            echo "Downloaded: $(ls -lh "$path")"
            return 0
        fi
        echo "Download failed, retrying in 10s..."
        sleep 10
    done
    echo "ERROR: Failed to download $url after $retries attempts"
    exit 1
}

for entry in "${MODELS[@]}"; do
    download_model "${entry%%|*}" "${entry#*|}"
done
MODELS_MAX=${#MODELS[@]}

# ── Dashboard auth ────────────────────────────────────────────────────────────
if [ -n "$DASHBOARD_KEY" ]; then
    HASH=$(openssl passwd -apr1 "$DASHBOARD_KEY")
    echo "admin:$HASH" > /tmp/.htpasswd
    echo "Dashboard auth: enabled (user=admin)"
else
    echo "WARNING: DASHBOARD_KEY not set, /dash-2e215f981f3f is unprotected"
    printf 'admin:$apr1$open$open\n' > /tmp/.htpasswd
fi

# ── nginx temp dirs ───────────────────────────────────────────────────────────
mkdir -p /tmp/nginx-{client-body,proxy,fastcgi,uwsgi,scgi}

# ── MCP servers (shared across all GPU backends) ─────────────────────────────
echo ""
echo "=== Starting MCP servers ==="

# Brave Search MCP (native HTTP) on 127.0.0.1:8001/mcp
if [ -n "$BRAVE_API_KEY" ]; then
    echo "Starting Brave Search MCP on 127.0.0.1:8001"
    brave-search-mcp-server --transport http --host 127.0.0.1 --port 8001 &
else
    echo "BRAVE_API_KEY not set -- skipping Brave Search MCP"
fi

# ArXiv MCP (stdio) bridged via supergateway on 127.0.0.1:8002/sse
echo "Starting ArXiv MCP on 127.0.0.1:8002 (via supergateway)"
supergateway --stdio "arxiv-mcp-server" --port 8002 --baseUrl http://127.0.0.1:8002 &

sleep 1

# ── Detect GPUs and start one llama-server per GPU ───────────────────────────
GPU_COUNT=$(nvidia-smi -L 2>/dev/null | wc -l || echo 1)
echo "GPUs detected: $GPU_COUNT"

BACKENDS=""
for i in $(seq 0 $((GPU_COUNT - 1))); do
    PORT=$((7861 + i))
    echo "Starting llama-server on GPU $i → port $PORT"
    CUDA_VISIBLE_DEVICES=$i /app/bin/llama-server \
        --models-dir "$MODEL_DIR" \
        --models-max "$MODELS_MAX" \
        --host 127.0.0.1 \
        --port "$PORT" \
        -ngl 99 \
        -fa on \
        -np 4 \
        -c 262144 \
        --metrics \
        --jinja \
        --webui-mcp-proxy \
        --temp 0.5 --top-p 0.85 --top-k 20 --min-p 0 \
        --reasoning-budget 0 --reasoning-format none \
        --chat-template-kwargs '{"enable_thinking": false}' \
        --log-disable &
    BACKENDS="$BACKENDS    server 127.0.0.1:$PORT;\n"
done

printf "upstream llama_backends {\n    least_conn;\n${BACKENDS}}\n" > /tmp/nginx-upstream.conf

# ── Write stub files so /gpu and /analytics never 404 before first tick ──────
echo '{"ts":null,"gpus":[]}' > /tmp/gpu-stats.json
echo '# waiting for first metrics scrape...' > /tmp/llama-metrics.txt
echo '{"updated_at":null,"summary_24h":{"requests":0,"unique_users":0},"summary_7d":{"requests":0,"unique_users":0},"summary_total":{"requests":0,"unique_users":0},"requests_by_hour":[],"requests_by_day":[],"top_users":[]}' > /tmp/analytics.json

# ── Start metrics pusher with watchdog ────────────────────────────────────────
start_metrics_pusher() {
    while true; do
        echo "[watchdog] Starting metrics_pusher.py..."
        python3 /app/metrics_pusher.py || true
        echo "[watchdog] metrics_pusher.py exited — restarting in 5s..."
        sleep 5
    done
}
start_metrics_pusher &

echo ""
echo "=== Bonsai-demo ==="
echo "  Models: $(ls "$MODEL_DIR"/*.gguf | xargs -n1 basename | tr '\n' ', ' | sed 's/,$//')"
echo "  GPUs:   $GPU_COUNT"
echo "  Port:   7860 (nginx → llama-server)"
echo ""

exec nginx -c /app/nginx.conf