27 lines
1.3 KiB
Bash
27 lines
1.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Wait for base run, then run: cache-dit plain (3tier) -> cache+lora (TP4x2 3tier)
|
|
set -Eeuo pipefail
|
|
LOGP=/data/wxy/sskj-h3/throughput
|
|
REC=/data/wxy/sskj-h3/throughput/sglang-lora/records_rewritten.json
|
|
|
|
echo "========== waiting for base tmux session to finish =========="
|
|
while tmux has-session -t base_newprompt 2>/dev/null; do sleep 60; done
|
|
echo "========== base done; verifying GPUs free =========="
|
|
while :; do
|
|
busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
|
|
[[ "$busy" -eq 0 ]] && break
|
|
echo "gpus still busy: $busy MiB; waiting"; sleep 60
|
|
done
|
|
echo "========== [2/3] CACHE-DIT plain (new prompt, TP2x4, 3 tiers) =========="
|
|
cd /data/wxy/sskj-h3/throughput/sglang-cache-dit
|
|
RECORDS=$REC RUN_ID="ref2va-feishu-cachedit-3tier-newprompt-$(date +%Y%m%d-%H%M%S)" \
|
|
bash scripts/run_sglang_h3_ref2va_feishu_cachedit_3tier_tp2x4.sh 2>&1 | tee $LOGP/cachedit_newprompt_run.log
|
|
echo "========== [3/3] CACHE+LORA (new prompt, TP4x2, 3 tiers) =========="
|
|
while :; do
|
|
busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
|
|
[[ "$busy" -eq 0 ]] && break
|
|
echo "gpus still busy: $busy; waiting"; sleep 60
|
|
done
|
|
bash /data/wxy/sskj-h3/throughput/run_cache_lora_tp4_3tier.sh 2>&1 | tee $LOGP/cache_lora_tp4_run.log
|
|
echo "========== PART2 COMPLETE =========="
|