Update README.md

README.md

@@ -196,18 +196,33 @@ We rely on [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-h
 <details>
 <summary> Reproduce Model Quality Results </summary>
 
+
+## language eval
 Need to install lm-eval from source:
 https://github.com/EleutherAI/lm-evaluation-harness#install
 
-## baseline
 ```Shell
-lm_eval --model hf --model_args pretrained=google/gemma-3-27b-it --tasks mmlu --device cuda:0 --batch_size 8
+export MODEL=google/gemma-3-27b-it # or pytorch/gemma-3-27b-it-INT4
+lm_eval --model hf --model_args pretrained=$MODEL --tasks mmlu --device cuda:0 --batch_size 8
 ```
 
-## INT4
+## multi-modal eval
+Need to install lmms-eval from source:
+`pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git`
+
 ```Shell
-export MODEL=pytorch/gemma-3-27b-it-INT4
-lm_eval --model hf --model_args pretrained=$MODEL --tasks mmlu --device cuda:0 --batch_size 8
+NUM_PROCESSES=8
+MAIN_PORT=12345
+MODEL_ID=google/gemma-3-27b-it # or pytorch/gemma-3-27b-it-INT4
+TASKS=chartqa  # or tasks from https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/models/simple
+BATCH_SIZE=1
+OUTPUT_PATH=./logs/
+
+accelerate launch --num_processes "${NUM_PROCESSES}" --main_process_port "${MAIN_PORT}" -m lmms_eval \
+  --model gemma3 \
+  --model_args "pretrained=${MODEL_ID}" \
+  --tasks "${TASKS}" \
+  --batch_size "${BATCH_SIZE}" --output_path "${OUTPUT_PATH}"
 ```
 </details>