hojo_benchmark.sh

#!/usr/bin/env bash
set -e

echo "============================================"
echo "HOJO ASR Full Benchmark: Original vs Improved"
echo "H200 GPU | FULL datasets (no sample limit)"
echo "============================================"
echo ""

cd /app
export PYTHONPATH=/app

# Copy both versions of run_eval.py from bucket
cp /results/hojo_original.py /app/run_eval_original.py
cp /results/hojo_improved.py /app/run_eval_improved.py

# Datasets to benchmark
DATASETS=(
  "librispeech test.clean"
  "librispeech test.other"
  "ami test"
  "common_voice test"
  "gigaspeech test"
  "earnings22 test"
  "voxpopuli test"
  "spgispeech test"
  "tedlium test"
)

RESULTS_DIR="/results"
RESULTS_FILE="$RESULTS_DIR/benchmark_results.csv"
echo "dataset,split,run,wer,rtfx,elapsed_s" > "$RESULTS_FILE"

for entry in "${DATASETS[@]}"; do
  DATASET=$(echo "$entry" | cut -d' ' -f1)
  SPLIT=$(echo "$entry" | cut -d' ' -f2)
  
  echo ""
  echo "=== $DATASET / $SPLIT ==="
  
  # Run 1: ORIGINAL code (no warmup, no max_eval_samples enforcement)
  echo "  [1/2] Running ORIGINAL code (full dataset)..."
  START=$(date +%s)
  OUTPUT_ORIG=$(python /app/run_eval_original.py \
    --model_id HojoAI/Hojo-ASR-V1 \
    --dataset "$DATASET" \
    --split "$SPLIT" \
    --device 0 \
    --batch_size 32 \
    2>&1) || true
  END=$(date +%s)
  ELAPSED_ORIG=$((END - START))
  
  WER_ORIG=$(echo "$OUTPUT_ORIG" | grep -oP "WER:\s*\K[0-9.]+" || echo "N/A")
  RTFX_ORIG=$(echo "$OUTPUT_ORIG" | grep -oP "RTFx:\s*\K[0-9.]+" || echo "N/A")
  echo "  ORIGINAL: WER=$WER_ORIG  RTFx=$RTFX_ORIG  (${ELAPSED_ORIG}s)"
  echo "$DATASET,$SPLIT,original,$WER_ORIG,$RTFX_ORIG,$ELAPSED_ORIG" >> "$RESULTS_FILE"
  
  # Run 2: IMPROVED code (with warmup=10, full dataset)
  echo "  [2/2] Running IMPROVED code (full dataset, warmup=10)..."
  START=$(date +%s)
  OUTPUT_IMP=$(python /app/run_eval_improved.py \
    --model_id HojoAI/Hojo-ASR-V1 \
    --dataset "$DATASET" \
    --split "$SPLIT" \
    --device 0 \
    --batch_size 32 \
    --warmup_steps 10 \
    2>&1) || true
  END=$(date +%s)
  ELAPSED_IMP=$((END - START))
  
  WER_IMP=$(echo "$OUTPUT_IMP" | grep -oP "WER:\s*\K[0-9.]+" || echo "N/A")
  RTFX_IMP=$(echo "$OUTPUT_IMP" | grep -oP "RTFx:\s*\K[0-9.]+" || echo "N/A")
  echo "  IMPROVED: WER=$WER_IMP  RTFx=$RTFX_IMP  (${ELAPSED_IMP}s)"
  echo "$DATASET,$SPLIT,improved,$WER_IMP,$RTFX_IMP,$ELAPSED_IMP" >> "$RESULTS_FILE"
  
  echo "  --- Comparison ---"
  echo "  WER:  original=$WER_ORIG  vs  improved=$WER_IMP"
  echo "  RTFx: original=$RTFX_ORIG  vs  improved=$RTFX_IMP"
done

echo ""
echo "============================================"
echo "Benchmark complete!"
echo "Results saved to: $RESULTS_FILE"
echo "============================================"
cat "$RESULTS_FILE"