32 lines
1.2 KiB
Bash
32 lines
1.2 KiB
Bash
#!/usr/bin/env bash
|
|
# 1) Larry LoRA base (9 grid points / 8 NFE, no cache, TP2x4)
|
|
# 2) Larry LoRA + Cache-DiT (9 grid / 8 NFE, TP4x2, 3 tiers)
|
|
set -Eeuo pipefail
|
|
LOGP=/data/wxy/sskj-h3/throughput
|
|
REC=/data/wxy/sskj-h3/throughput/sglang-lora/records_rewritten.json
|
|
|
|
say() { echo "[$(date +%F_%T)] $*"; }
|
|
wait_gpu_free() {
|
|
say "waiting for GPUs to be free..."
|
|
while :; do
|
|
busy=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader | awk "{s+=\$1} END{print s+0}")
|
|
[[ "$busy" -eq 0 ]] && break
|
|
echo "gpus busy: $busy MiB"; sleep 60
|
|
done
|
|
say "GPUs free"
|
|
}
|
|
|
|
echo "########## [1/2] LARRY LORA (no cache, TP2x4, 8 NFE) ##########"
|
|
wait_gpu_free
|
|
cd /data/wxy/sskj-h3/throughput/sglang-lora
|
|
RECORDS=$REC RUN_ID="ref2va-feishu-larry-newprompt-tp2x4-8nfe-$(date +%Y%m%d-%H%M%S)" \
|
|
bash scripts/run_sglang_h3_ref2va_feishu_larry_v4_600_tp2x4.sh 2>&1 | tee $LOGP/larry_newprompt_run.log
|
|
|
|
echo "########## [2/2] CACHE+LORA (TP4x2, 3 tiers, 8 NFE) ##########"
|
|
wait_gpu_free
|
|
cd /data/wxy/sskj-h3/throughput
|
|
RUN_ID="ref2va-feishu-cache-lora-tp4x2-3tier-8nfe-$(date +%Y%m%d-%H%M%S)" \
|
|
bash run_cache_lora_tp4_3tier.sh 2>&1 | tee $LOGP/cache_lora_tp4_8nfe_run.log
|
|
|
|
echo "########## ALL DONE ##########"
|