#!/usr/bin/env bash
# 1) Larry LoRA base (9 grid points / 8 NFE, no cache, TP2x4)
# 2) Larry LoRA + Cache-DiT (9 grid / 8 NFE, TP4x2, 3 tiers)
set -Eeuo pipefail
LOGP=/data/wxy/sskj-h3/throughput
REC=/data/wxy/sskj-h3/throughput/sglang-lora/records_rewritten.json

say() { echo "[$(date +%F_%T)] $*"; }
wait_gpu_free() {
  say "waiting for GPUs to be free..."
  while :; do
    busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
    [[ "$busy" -eq 0 ]] && break
    echo "gpus busy: $busy MiB"; sleep 60
  done
  say "GPUs free"
}

echo "########## [1/2] LARRY LORA (no cache, TP2x4, 8 NFE) ##########"
wait_gpu_free
cd /data/wxy/sskj-h3/throughput/sglang-lora
RECORDS=$REC RUN_ID="ref2va-feishu-larry-newprompt-tp2x4-8nfe-$(date +%Y%m%d-%H%M%S)" \
  bash scripts/run_sglang_h3_ref2va_feishu_larry_v4_600_tp2x4.sh 2>&1 | tee $LOGP/larry_newprompt_run.log

echo "########## [2/2] CACHE+LORA (TP4x2, 3 tiers, 8 NFE) ##########"
wait_gpu_free
cd /data/wxy/sskj-h3/throughput
RUN_ID="ref2va-feishu-cache-lora-tp4x2-3tier-8nfe-$(date +%Y%m%d-%H%M%S)" \
  bash run_cache_lora_tp4_3tier.sh 2>&1 | tee $LOGP/cache_lora_tp4_8nfe_run.log

echo "########## ALL DONE ##########"