#!/usr/bin/env bash
# Wait for base run, then run: cache-dit plain (3tier) -> cache+lora (TP4x2 3tier)
set -Eeuo pipefail
LOGP=/data/wxy/sskj-h3/throughput
REC=/data/wxy/sskj-h3/throughput/sglang-lora/records_rewritten.json

echo "========== waiting for base tmux session to finish =========="
while tmux has-session -t base_newprompt 2>/dev/null; do sleep 60; done
echo "========== base done; verifying GPUs free =========="
while :; do
  busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
  [[ "$busy" -eq 0 ]] && break
  echo "gpus still busy: $busy MiB; waiting"; sleep 60
done
echo "========== [2/3] CACHE-DIT plain (new prompt, TP2x4, 3 tiers) =========="
cd /data/wxy/sskj-h3/throughput/sglang-cache-dit
RECORDS=$REC RUN_ID="ref2va-feishu-cachedit-3tier-newprompt-$(date +%Y%m%d-%H%M%S)" \
  bash scripts/run_sglang_h3_ref2va_feishu_cachedit_3tier_tp2x4.sh 2>&1 | tee $LOGP/cachedit_newprompt_run.log
echo "========== [3/3] CACHE+LORA (new prompt, TP4x2, 3 tiers) =========="
while :; do
  busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
  [[ "$busy" -eq 0 ]] && break
  echo "gpus still busy: $busy; waiting"; sleep 60
done
bash /data/wxy/sskj-h3/throughput/run_cache_lora_tp4_3tier.sh 2>&1 | tee $LOGP/cache_lora_tp4_run.log
echo "========== PART2 COMPLETE =========="