353 lines
20 KiB
Bash
Executable File
353 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Full MiniMax-H3/SGLang profile matrix for one 8x RTX 6000D host.
|
|
# Owns only process groups it starts; refuses occupied ports and supports resume.
|
|
set -Eeuo pipefail
|
|
|
|
ROOT=${ROOT:-/data/wxy}
|
|
MODEL=${MODEL:-/data/hf_models/MiniMax-H3}
|
|
PYTHON=${PYTHON:-/root/.miniconda3/envs/sglang/bin/python}
|
|
SGLANG=${SGLANG:-/root/.miniconda3/envs/sglang/bin/sglang}
|
|
CLIENT=${CLIENT:-$ROOT/h3_profile/h3_profile_client.py}
|
|
ANALYZER=${ANALYZER:-$ROOT/h3_profile/h3_profile_analyze.py}
|
|
RUN_ID=${RUN_ID:-h3-profile-$(date +%Y%m%d-%H%M%S)}
|
|
RESULT_ROOT=${RESULT_ROOT:-$ROOT/profile_results/$RUN_ID}
|
|
MODE=${MODE:-full} # light | quick_matrix | quick_ref_matrix | targeted_profile | smoke | baseline | full
|
|
STEPS=${STEPS:-20}
|
|
DURATION=${DURATION:-5}
|
|
REPEATS=${REPEATS:-1}
|
|
WARMUP=${WARMUP:-1}
|
|
WARMUP_STEPS=${WARMUP_STEPS:-5}
|
|
ENABLE_TORCH_PROFILE=${ENABLE_TORCH_PROFILE:-1}
|
|
ENABLE_NSYS=${ENABLE_NSYS:-1}
|
|
ENABLE_NCU=${ENABLE_NCU:-0} # expensive; requires NCU_KERNEL_REGEX
|
|
NCU_KERNEL_REGEX=${NCU_KERNEL_REGEX:-}
|
|
NSYS_DELAY_SECONDS=${NSYS_DELAY_SECONDS:-70}
|
|
NSYS_DURATION_SECONDS=${NSYS_DURATION_SECONDS:-1800}
|
|
LIGHT_SCENARIO=${LIGHT_SCENARIO:-RVA}
|
|
LIGHT_RESIDENT_INSTANCES=${LIGHT_RESIDENT_INSTANCES:-4} # 1, 2, or 4
|
|
LIGHT_ACTIVE_INSTANCES=${LIGHT_ACTIVE_INSTANCES:-1}
|
|
LIGHT_GPU_OFFSET=${LIGHT_GPU_OFFSET:-0}
|
|
LIGHT_TORCH_PROFILE=${LIGHT_TORCH_PROFILE:-0}
|
|
LIGHT_SAMPLES=${LIGHT_SAMPLES:-5}
|
|
BASE_PORT=${BASE_PORT:-30010}
|
|
PORT_STRIDE=${PORT_STRIDE:-10}
|
|
MASTER_PORT_BASE=${MASTER_PORT_BASE:-31000}
|
|
SCHEDULER_PORT_BASE=${SCHEDULER_PORT_BASE:-32000}
|
|
SERVER_START_TIMEOUT=${SERVER_START_TIMEOUT:-1800}
|
|
REFERENCE_IMAGE=${REFERENCE_IMAGE:-$ROOT/sskj-MiniMax-H3/assets/reference_images/landscape_mountain_lake.jpg}
|
|
REFERENCE_IMAGES_DIR=${REFERENCE_IMAGES_DIR:-$ROOT/h3_profile/assets/reference_images_5}
|
|
REFERENCE_VIDEO_1S=${REFERENCE_VIDEO_1S:-$MODEL/assets/r2va.mp4}
|
|
REFERENCE_VIDEO_5S=${REFERENCE_VIDEO_5S:-$MODEL/assets/ref2va.mp4}
|
|
REFERENCE_VIDEO_10S=${REFERENCE_VIDEO_10S:-$MODEL/assets/h3_direct_768p.mp4}
|
|
NSYS=${NSYS:-/usr/local/cuda-13.2/bin/nsys}
|
|
NCU=${NCU:-/usr/local/cuda-13.2/bin/ncu}
|
|
MEDIA_BIN_DIR=${MEDIA_BIN_DIR:-/root/.miniconda3/envs/deploy/bin}
|
|
|
|
declare -a SERVER_PIDS=() MONITOR_PIDS=() CLIENT_PIDS=()
|
|
log(){ printf '[%s] %s\n' "$(date '+%F %T')" "$*" | tee -a "$RESULT_ROOT/run.log"; }
|
|
die(){ log "ERROR: $*"; exit 1; }
|
|
|
|
mkdir -p "$RESULT_ROOT"/{metadata,phases,system,torch,nsys,ncu}
|
|
[[ -x "$PYTHON" && -x "$SGLANG" && -f "$CLIENT" && -f "$ANALYZER" ]] || die "missing executable/client/analyzer"
|
|
|
|
port_open(){ "$PYTHON" - "$1" <<'PY'
|
|
import socket,sys
|
|
s=socket.socket(); s.settimeout(.3)
|
|
try: s.connect(("127.0.0.1",int(sys.argv[1])))
|
|
except OSError: raise SystemExit(1)
|
|
raise SystemExit(0)
|
|
PY
|
|
}
|
|
|
|
stop_monitors(){ local p; for p in "${MONITOR_PIDS[@]:-}"; do kill -TERM "$p" 2>/dev/null || true; done; MONITOR_PIDS=(); }
|
|
stop_servers(){
|
|
local p deadline alive
|
|
for p in "${SERVER_PIDS[@]:-}"; do kill -INT -- "-$p" 2>/dev/null || kill -INT "$p" 2>/dev/null || true; done
|
|
deadline=$((SECONDS+120))
|
|
while ((SECONDS<deadline)); do alive=0; for p in "${SERVER_PIDS[@]:-}"; do kill -0 "$p" 2>/dev/null && alive=1; done; ((alive==0)) && break; sleep 2; done
|
|
for p in "${SERVER_PIDS[@]:-}"; do
|
|
if kill -0 "$p" 2>/dev/null; then kill -TERM -- "-$p" 2>/dev/null || true; sleep 3; fi
|
|
if kill -0 "$p" 2>/dev/null; then kill -KILL -- "-$p" 2>/dev/null || true; fi
|
|
wait "$p" 2>/dev/null || true
|
|
done
|
|
SERVER_PIDS=()
|
|
}
|
|
cleanup(){ local rc=$? p; trap - EXIT INT TERM; for p in "${CLIENT_PIDS[@]:-}"; do kill -TERM "$p" 2>/dev/null || true; done; stop_monitors; stop_servers; exit "$rc"; }
|
|
trap cleanup EXIT INT TERM
|
|
|
|
collect_metadata(){
|
|
date -Ins > "$RESULT_ROOT/metadata/date.txt"
|
|
hostname > "$RESULT_ROOT/metadata/hostname.txt"
|
|
uname -a > "$RESULT_ROOT/metadata/uname.txt"
|
|
nvidia-smi -q > "$RESULT_ROOT/metadata/nvidia-smi-q.txt"
|
|
nvidia-smi topo -m > "$RESULT_ROOT/metadata/nvidia-topology.txt"
|
|
nvidia-smi --query-gpu=index,name,uuid,memory.total,pci.bus_id,driver_version --format=csv > "$RESULT_ROOT/metadata/gpus.csv"
|
|
lscpu > "$RESULT_ROOT/metadata/lscpu.txt"
|
|
numactl --hardware > "$RESULT_ROOT/metadata/numa.txt" 2>&1 || true
|
|
"$PYTHON" -m pip freeze > "$RESULT_ROOT/metadata/pip-freeze.txt"
|
|
"$PYTHON" - <<'PY' > "$RESULT_ROOT/metadata/torch.json"
|
|
import json,torch
|
|
print(json.dumps({"torch":torch.__version__,"cuda":torch.version.cuda,"cudnn":torch.backends.cudnn.version()},indent=2))
|
|
PY
|
|
git -C "$MODEL" rev-parse HEAD > "$RESULT_ROOT/metadata/model-revision.txt" 2>/dev/null || true
|
|
cp "$0" "$RESULT_ROOT/metadata/"
|
|
cp "$CLIENT" "$ANALYZER" "$RESULT_ROOT/metadata/"
|
|
}
|
|
|
|
start_monitors(){
|
|
local dir=$1; mkdir -p "$dir"
|
|
nvidia-smi dmon -s pucvmet -d 1 -o DT > "$dir/nvidia-dmon.log" 2>&1 & MONITOR_PIDS+=("$!")
|
|
dcgmi dmon -e 1001,1002,1003,1004,1005,1009,1010,1011,1012,1016,1017,1018 -d 1000 > "$dir/dcgm-dmon.log" 2>&1 & MONITOR_PIDS+=("$!")
|
|
pidstat -durh 1 > "$dir/pidstat.log" 2>&1 & MONITOR_PIDS+=("$!")
|
|
iostat -xz 1 > "$dir/iostat.log" 2>&1 & MONITOR_PIDS+=("$!")
|
|
vmstat 1 > "$dir/vmstat.log" 2>&1 & MONITOR_PIDS+=("$!")
|
|
}
|
|
|
|
wait_healthy(){
|
|
local port=$1 pid=$2 logfile=$3 deadline=$((SECONDS+SERVER_START_TIMEOUT))
|
|
while ((SECONDS<deadline)); do
|
|
curl -fsS --max-time 5 "http://127.0.0.1:$port/health" >/dev/null 2>&1 && return 0
|
|
if ! kill -0 "$pid" 2>/dev/null; then tail -100 "$logfile"; return 1; fi
|
|
sleep 5
|
|
done
|
|
tail -100 "$logfile"; return 1
|
|
}
|
|
|
|
start_server(){
|
|
local variant=$1 tp=$2 replica=$3 gpus=$4 phase=$5 profiler=${6:-none}
|
|
local port=$((BASE_PORT+replica*PORT_STRIDE)) master=$((MASTER_PORT_BASE+replica*PORT_STRIDE)) sched=$((SCHEDULER_PORT_BASE+replica*PORT_STRIDE))
|
|
local dir="$phase/server_${replica}_port${port}" logf="$phase/server_${replica}_port${port}/server.log"
|
|
local stage_logging=1 stage_sync=1
|
|
[[ "$profiler" == baseline ]] && { stage_logging=0; stage_sync=0; }
|
|
[[ "$profiler" == nsys || "$profiler" == ncu ]] && stage_sync=0
|
|
mkdir -p "$dir/outputs" "$dir/torch"
|
|
for p in "$port" "$((port+1))" "$master" "$sched"; do port_open "$p" && die "port occupied: $p"; done
|
|
local -a cmd=("$SGLANG" serve --model-path "$MODEL" --model-variant "$variant" --backend sglang --performance-mode speed
|
|
--num-gpus "$tp" --tp-size "$tp" --ulysses-degree 1 --use-fsdp-inference false --enable-torch-compile false
|
|
--batching-max-size 1 --batching-delay-ms 0 --enable-layerwise-nvtx-marker --host 0.0.0.0 --port "$port"
|
|
--master-port "$master" --scheduler-port "$sched" --output-path "$dir/outputs" --log-requests --log-requests-level 2)
|
|
local -a wrapper=(setsid)
|
|
if [[ "$profiler" == nsys && -x "$NSYS" ]]; then
|
|
wrapper=(setsid "$NSYS" profile --force-overwrite=true --delay "$NSYS_DELAY_SECONDS" --duration "$NSYS_DURATION_SECONDS"
|
|
--trace=cuda,nvtx,nccl,osrt --sample=none --cpuctxsw=none --cuda-graph-trace=node -o "$dir/server")
|
|
elif [[ "$profiler" == ncu && -x "$NCU" && -n "$NCU_KERNEL_REGEX" ]]; then
|
|
wrapper=(setsid "$NCU" --target-processes all --kernel-name-base demangled --kernel-name "regex:$NCU_KERNEL_REGEX" --launch-skip 5 --launch-count 3 --set full -o "$dir/server")
|
|
fi
|
|
log "start variant=$variant tp=$tp replica=$replica GPUs=$gpus port=$port profiler=$profiler"
|
|
CUDA_VISIBLE_DEVICES="$gpus" PATH="$MEDIA_BIN_DIR:$PATH" PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false SGLANG_USE_RUNAI_MODEL_STREAMER=false \
|
|
SGLANG_DIFFUSION_STAGE_LOGGING="$stage_logging" SGLANG_DIFFUSION_SYNC_STAGE_PROFILING="$stage_sync" \
|
|
SGLANG_DIFFUSION_TORCH_PROFILER_DIR="$dir/torch" NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=COLL \
|
|
"${wrapper[@]}" "${cmd[@]}" > "$logf" 2>&1 &
|
|
SERVER_PIDS+=("$!")
|
|
}
|
|
|
|
start_layout(){
|
|
local layout=$1 phase=$2 profiler=${3:-none}; SERVER_PIDS=()
|
|
case "$layout" in
|
|
fl_tp2_1) start_server FL2VA 2 0 0,1 "$phase" "$profiler";;
|
|
ref_tp2_1) start_server Ref2VA 2 0 0,1 "$phase" "$profiler";;
|
|
fl_tp2_1_high) start_server FL2VA 2 0 4,5 "$phase" "$profiler";;
|
|
ref_tp2_1_high) start_server Ref2VA 2 0 4,5 "$phase" "$profiler";;
|
|
fl_tp2_2_high) start_server FL2VA 2 0 4,5 "$phase" "$profiler"; start_server FL2VA 2 1 6,7 "$phase" "$profiler";;
|
|
ref_tp2_2_high) start_server Ref2VA 2 0 4,5 "$phase" "$profiler"; start_server Ref2VA 2 1 6,7 "$phase" "$profiler";;
|
|
fl_tp2_4) for r in 0 1 2 3; do start_server FL2VA 2 "$r" "$((r*2)),$((r*2+1))" "$phase" "$profiler"; done;;
|
|
ref_tp2_4) for r in 0 1 2 3; do start_server Ref2VA 2 "$r" "$((r*2)),$((r*2+1))" "$phase" "$profiler"; done;;
|
|
mixed_tp2_4)
|
|
start_server FL2VA 2 0 0,1 "$phase" "$profiler"; start_server FL2VA 2 1 2,3 "$phase" "$profiler"
|
|
start_server Ref2VA 2 2 4,5 "$phase" "$profiler"; start_server Ref2VA 2 3 6,7 "$phase" "$profiler";;
|
|
fl_tp8_1) start_server FL2VA 8 0 0,1,2,3,4,5,6,7 "$phase" "$profiler";;
|
|
ref_tp8_1) start_server Ref2VA 8 0 0,1,2,3,4,5,6,7 "$phase" "$profiler";;
|
|
*) die "unknown layout $layout";;
|
|
esac
|
|
local r port logfile
|
|
for ((r=0;r<${#SERVER_PIDS[@]};r++)); do port=$((BASE_PORT+r*PORT_STRIDE)); logfile="$phase/server_${r}_port${port}/server.log"; wait_healthy "$port" "${SERVER_PIDS[$r]}" "$logfile" || die "server failed: $logfile"; done
|
|
}
|
|
|
|
run_client(){
|
|
local deployment=$1 scenario=$2 replica=$3 phase=$4 profile=${5:-none} warmup_override=${6:-$WARMUP} repeats_override=${7:-$REPEATS}
|
|
local port=$((BASE_PORT+replica*PORT_STRIDE)) out="$phase/client_${replica}_${scenario}"; mkdir -p "$out"
|
|
local -a flags=()
|
|
[[ "$profile" == denoise || "$profile" == all ]] && flags+=(--profile --num-profiled-timesteps 5)
|
|
[[ "$profile" == all ]] && flags+=(--profile-all-stages)
|
|
"$PYTHON" "$CLIENT" --deployment "$deployment" --scenario "$scenario" --port "$port" --replica-index "$replica" \
|
|
--model "$MODEL" --reference-image "$REFERENCE_IMAGE" --reference-images-dir "$REFERENCE_IMAGES_DIR" --reference-video-1s "$REFERENCE_VIDEO_1S" \
|
|
--reference-video-5s "$REFERENCE_VIDEO_5S" --reference-video-10s "$REFERENCE_VIDEO_10S" \
|
|
--steps "$STEPS" --duration "$DURATION" --short-edge 768 --aspect-ratio 16:9 --seed 1101 \
|
|
--repeats "$repeats_override" --warmup "$warmup_override" --warmup-steps "$WARMUP_STEPS" --perf-dir "$out/perf" --output "$out/results.jsonl" "${flags[@]}" \
|
|
> "$out/client.log" 2>&1
|
|
}
|
|
|
|
run_light_distributed_phase(){
|
|
local name=$1 layout=$2 deployment=$3 scenario=$4 active=$5
|
|
local phase="$RESULT_ROOT/phases/$name"; mkdir -p "$phase"
|
|
[[ -f "$phase/DONE" ]] && { log "resume skip $name"; return; }
|
|
((active >= 1 && active <= LIGHT_RESIDENT_INSTANCES)) || die "invalid LIGHT_ACTIVE_INSTANCES=$active"
|
|
log "===== distributed light phase $name: samples=$LIGHT_SAMPLES active_instances=$active ====="
|
|
start_monitors "$phase/system"; start_layout "$layout" "$phase" none
|
|
local base=$((LIGHT_SAMPLES/active)) remainder=$((LIGHT_SAMPLES%active)) r count failed=0
|
|
CLIENT_PIDS=()
|
|
for ((r=0;r<active;r++)); do
|
|
count=$base; ((r<remainder)) && count=$((count+1))
|
|
((count>0)) || continue
|
|
run_client "$deployment" "$scenario" "$r" "$phase" none 0 "$count" & CLIENT_PIDS+=("$!")
|
|
done
|
|
for p in "${CLIENT_PIDS[@]}"; do wait "$p" || failed=1; done; CLIENT_PIDS=()
|
|
stop_monitors; stop_servers
|
|
((failed==0)) || die "distributed light phase $name had failed client(s)"
|
|
touch "$phase/DONE"
|
|
}
|
|
|
|
run_phase_serial(){
|
|
local name=$1 layout=$2 deployment=$3 profiler=$4; shift 4
|
|
local phase="$RESULT_ROOT/phases/$name"; mkdir -p "$phase"
|
|
if [[ -f "$phase/DONE" ]]; then log "resume skip $name"; return; fi
|
|
log "===== serial phase $name ====="; start_monitors "$phase/system"; start_layout "$layout" "$phase" "$profiler"
|
|
local item scenario replica first=1 failed=0
|
|
for item in "$@"; do
|
|
scenario=${item%%:*}; replica=${item##*:}
|
|
if ((first)); then run_client "$deployment" "$scenario" "$replica" "$phase" none "$WARMUP" || failed=1; first=0
|
|
else run_client "$deployment" "$scenario" "$replica" "$phase" none 0 || failed=1
|
|
fi
|
|
done
|
|
stop_monitors; stop_servers
|
|
((failed==0)) || die "phase $name had failed client(s)"
|
|
touch "$phase/DONE"
|
|
}
|
|
|
|
run_phase(){
|
|
local name=$1 layout=$2 deployment=$3 profiler=$4; shift 4
|
|
local phase="$RESULT_ROOT/phases/$name"; mkdir -p "$phase"
|
|
if [[ -f "$phase/DONE" ]]; then log "resume skip $name"; return; fi
|
|
log "===== phase $name ====="; start_monitors "$phase/system"; start_layout "$layout" "$phase" "$profiler"
|
|
local item scenario replica; CLIENT_PIDS=()
|
|
for item in "$@"; do scenario=${item%%:*}; replica=${item##*:}; run_client "$deployment" "$scenario" "$replica" "$phase" none & CLIENT_PIDS+=("$!"); done
|
|
local failed=0; for p in "${CLIENT_PIDS[@]}"; do wait "$p" || failed=1; done; CLIENT_PIDS=()
|
|
stop_monitors; stop_servers
|
|
((failed==0)) || die "phase $name had failed client(s)"
|
|
touch "$phase/DONE"
|
|
}
|
|
|
|
run_torch_phase(){
|
|
local name=$1 layout=$2 deployment=$3 profile_kind=$4; shift 4
|
|
local phase="$RESULT_ROOT/phases/$name"; mkdir -p "$phase"
|
|
[[ -f "$phase/DONE" ]] && { log "resume skip $name"; return; }
|
|
log "===== torch phase $name ====="; start_monitors "$phase/system"; start_layout "$layout" "$phase" none
|
|
local scenario first=1 failed=0
|
|
for scenario in "$@"; do
|
|
if ((first)); then run_client "$deployment" "$scenario" 0 "$phase" "$profile_kind" "$WARMUP" || failed=1; first=0
|
|
else run_client "$deployment" "$scenario" 0 "$phase" "$profile_kind" 0 || failed=1
|
|
fi
|
|
done
|
|
stop_monitors; stop_servers; touch "$phase/DONE"
|
|
((failed==0)) || die "torch phase $name had failed client(s)"
|
|
}
|
|
|
|
run_nsys_phase(){
|
|
local name=$1 layout=$2 deployment=$3 scenario=$4
|
|
local phase="$RESULT_ROOT/phases/$name"; mkdir -p "$phase"
|
|
[[ -f "$phase/DONE" ]] && { log "resume skip $name"; return; }
|
|
[[ -x "$NSYS" ]] || { printf 'nsys missing: %s\n' "$NSYS" > "$phase/SKIPPED"; return; }
|
|
log "===== nsys phase $name ====="; start_monitors "$phase/system"; start_layout "$layout" "$phase" nsys
|
|
local r; CLIENT_PIDS=(); if [[ "$layout" == *_4 ]]; then for r in 0 1 2 3; do run_client "$deployment" "$scenario" "$r" "$phase" none & CLIENT_PIDS+=("$!"); done; else run_client "$deployment" "$scenario" 0 "$phase" none & CLIENT_PIDS+=("$!"); fi
|
|
local failed=0; for p in "${CLIENT_PIDS[@]}"; do wait "$p" || failed=1; done; CLIENT_PIDS=()
|
|
stop_monitors; stop_servers; ((failed==0)) || die "nsys phase failed"; touch "$phase/DONE"
|
|
}
|
|
|
|
collect_metadata
|
|
if [[ "$MODE" == light && "$LIGHT_GPU_OFFSET" == 4 && "$LIGHT_RESIDENT_INSTANCES" =~ ^[12]$ ]]; then
|
|
selected_gpu_csv=4,5
|
|
[[ "$LIGHT_RESIDENT_INSTANCES" == 2 ]] && selected_gpu_csv=4,5,6,7
|
|
if nvidia-smi -i "$selected_gpu_csv" --query-compute-apps=pid --format=csv,noheader | grep -q '[0-9]'; then
|
|
die "selected GPUs $selected_gpu_csv already have compute processes"
|
|
fi
|
|
else
|
|
if nvidia-smi --query-compute-apps=pid --format=csv,noheader | grep -q '[0-9]'; then die "GPU processes already running; refusing to interfere"; fi
|
|
fi
|
|
|
|
if [[ "$MODE" == targeted_profile ]]; then
|
|
REPEATS=1; WARMUP=0
|
|
run_torch_phase targeted_torch_all_F3 fl_tp2_1 TARGET_TORCH_TP2 all F3
|
|
run_torch_phase targeted_torch_all_RVA ref_tp2_1 TARGET_TORCH_TP2 all RVA_EMBEDDED
|
|
run_nsys_phase targeted_nsys_F3 fl_tp2_1 TARGET_NSYS_TP2 F3
|
|
run_nsys_phase targeted_nsys_RVA ref_tp2_1 TARGET_NSYS_TP2 RVA_EMBEDDED
|
|
elif [[ "$MODE" == quick_ref_matrix ]]; then
|
|
REPEATS=1; WARMUP=0
|
|
run_phase quick_matrix_ref_aligned ref_tp2_4 QUICK_REF2VA_ALIGNED_4scenarios none R1:0 R5:1 RV5:2 RVA_EMBEDDED:3
|
|
elif [[ "$MODE" == quick_matrix ]]; then
|
|
REPEATS=1; WARMUP=0
|
|
run_phase quick_matrix_fl fl_tp2_4 QUICK_FL2VA_4scenarios none F0:0 F1:1 F2:2 F3:3
|
|
run_phase quick_matrix_ref ref_tp2_4 QUICK_REF2VA_4scenarios none R1:0 R5:1 RV5:2 RVA_EMBEDDED:3
|
|
elif [[ "$MODE" == light ]]; then
|
|
# A small measured sample set. SGLang's built-in one-step server warmup
|
|
# remains enabled, but no additional client warmup is generated.
|
|
[[ "$LIGHT_SAMPLES" =~ ^[1-9][0-9]*$ ]] || die "LIGHT_SAMPLES must be a positive integer"
|
|
REPEATS="$LIGHT_SAMPLES"; WARMUP=0
|
|
case "$LIGHT_SCENARIO" in F0|F1|F2|F3) light_variant=fl;; *) light_variant=ref;; esac
|
|
[[ "$LIGHT_RESIDENT_INSTANCES" == 1 || "$LIGHT_RESIDENT_INSTANCES" == 2 || "$LIGHT_RESIDENT_INSTANCES" == 4 ]] \
|
|
|| die "LIGHT_RESIDENT_INSTANCES must be 1, 2, or 4"
|
|
[[ "$LIGHT_ACTIVE_INSTANCES" =~ ^[1-4]$ ]] || die "LIGHT_ACTIVE_INSTANCES must be 1..4"
|
|
((LIGHT_ACTIVE_INSTANCES <= LIGHT_RESIDENT_INSTANCES)) || die "active instances cannot exceed resident instances"
|
|
if [[ "$LIGHT_RESIDENT_INSTANCES" == 4 ]]; then
|
|
[[ "$LIGHT_GPU_OFFSET" == 0 ]] || die "four TP2 instances require LIGHT_GPU_OFFSET=0"
|
|
light_layout="${light_variant}_tp2_4"; light_deployment=LIGHT_tp2x4_one_active
|
|
elif [[ "$LIGHT_RESIDENT_INSTANCES" == 2 ]]; then
|
|
[[ "$LIGHT_GPU_OFFSET" == 4 ]] || die "two-instance partial-host mode currently requires LIGHT_GPU_OFFSET=4"
|
|
light_layout="${light_variant}_tp2_2_high"; light_deployment=LIGHT_tp2x2_one_active_gpu4_7
|
|
elif [[ "$LIGHT_GPU_OFFSET" == 4 ]]; then
|
|
light_layout="${light_variant}_tp2_1_high"; light_deployment=LIGHT_tp2_single_gpu4_5
|
|
else
|
|
light_layout="${light_variant}_tp2_1"; light_deployment=LIGHT_tp2_single
|
|
fi
|
|
if [[ "$LIGHT_TORCH_PROFILE" == 1 ]]; then
|
|
[[ "$LIGHT_ACTIVE_INSTANCES" == 1 ]] || die "Torch light profile supports exactly one active instance"
|
|
run_torch_phase "light_${LIGHT_SCENARIO}_torch_all" "$light_layout" "$light_deployment" all "$LIGHT_SCENARIO"
|
|
elif ((LIGHT_ACTIVE_INSTANCES > 1)); then
|
|
run_light_distributed_phase "light_${LIGHT_SCENARIO}_${LIGHT_ACTIVE_INSTANCES}active" "$light_layout" "LIGHT_tp2x${LIGHT_RESIDENT_INSTANCES}_${LIGHT_ACTIVE_INSTANCES}active" "$LIGHT_SCENARIO" "$LIGHT_ACTIVE_INSTANCES"
|
|
else
|
|
run_phase_serial "light_${LIGHT_SCENARIO}_stage" "$light_layout" "$light_deployment" none "$LIGHT_SCENARIO:0"
|
|
fi
|
|
elif [[ "$MODE" == smoke ]]; then
|
|
REPEATS=1; WARMUP=0; STEPS=${SMOKE_STEPS:-2}
|
|
run_phase_serial smoke_fl fl_tp2_1 P1_tp2_single none F3:0
|
|
run_phase_serial smoke_ref ref_tp2_1 P1_tp2_single none RVA:0
|
|
else
|
|
# Clean E2E controls: no stage synchronization and no request profiler.
|
|
run_phase_serial e2e_p1_fl fl_tp2_1 E2E_P1_tp2 baseline F3:0
|
|
run_phase_serial e2e_p1_ref ref_tp2_1 E2E_P1_tp2 baseline RVA:0
|
|
run_phase e2e_p3_fl fl_tp2_4 E2E_P3_tp2x4 baseline F3:0 F3:1 F3:2 F3:3
|
|
run_phase e2e_p4_ref ref_tp2_4 E2E_P4_tp2x4 baseline RVA:0 RVA:1 RVA:2 RVA:3
|
|
# Stage-attribution matrix: synchronized boundaries for accurate stage shares.
|
|
run_phase_serial p1_fl fl_tp2_1 P1_tp2_single none F0:0 F1:0 F2:0 F3:0
|
|
run_phase_serial p1_ref ref_tp2_1 P1_tp2_single none R1:0 R5:0 R6:0 R9:0 RV1:0 RV5:0 RV10:0 RIA:0 RVA:0 RVA_EMBEDDED:0 RMIX:0
|
|
if [[ "$MODE" == full ]]; then
|
|
run_phase p2_fl_resident fl_tp2_4 P2_tp2x4_one_active none F3:0
|
|
run_phase p2_ref_resident ref_tp2_4 P2_tp2x4_one_active none RVA:0
|
|
run_phase p3_fl_all fl_tp2_4 P3_tp2x4_fl_active none F3:0 F3:1 F3:2 F3:3
|
|
run_phase p4_ref_all ref_tp2_4 P4_tp2x4_ref_active none RVA:0 RVA:1 RVA:2 RVA:3
|
|
run_phase p5_mixed mixed_tp2_4 P5_tp2x4_mixed none F3:0 F3:1 RVA:2 RVA:3
|
|
run_phase p6_fl_tp8 fl_tp8_1 P6_tp8_single none F3:0
|
|
run_phase p6_ref_tp8 ref_tp8_1 P6_tp8_single none RVA:0
|
|
if [[ "$ENABLE_TORCH_PROFILE" == 1 ]]; then
|
|
run_torch_phase torch_denoise_fl fl_tp2_1 Torch_tp2 denoise F0 F3
|
|
run_torch_phase torch_denoise_ref ref_tp2_1 Torch_tp2 denoise R1 RVA RMIX
|
|
run_torch_phase torch_all_fl fl_tp2_1 TorchAll_tp2 all F3
|
|
run_torch_phase torch_all_ref ref_tp2_1 TorchAll_tp2 all RVA RMIX
|
|
fi
|
|
if [[ "$ENABLE_NSYS" == 1 ]]; then
|
|
run_nsys_phase nsys_F3_tp2 fl_tp2_1 Nsys_tp2 F3
|
|
run_nsys_phase nsys_RVA_tp2 ref_tp2_1 Nsys_tp2 RVA
|
|
run_nsys_phase nsys_F3_tp2x4 fl_tp2_4 Nsys_tp2x4 F3
|
|
run_nsys_phase nsys_RVA_tp2x4 ref_tp2_4 Nsys_tp2x4 RVA
|
|
fi
|
|
if [[ "$ENABLE_NCU" == 1 ]]; then
|
|
[[ -n "$NCU_KERNEL_REGEX" ]] || die "ENABLE_NCU=1 requires NCU_KERNEL_REGEX derived from Torch/Nsight top kernels"
|
|
# NCU is intentionally a second pass after kernel attribution.
|
|
phase="$RESULT_ROOT/phases/ncu_target"; start_monitors "$phase/system"; start_layout ref_tp2_1 "$phase" ncu; run_client NCU_tp2 RVA 0 "$phase" none; stop_monitors; stop_servers; touch "$phase/DONE"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
"$PYTHON" "$ANALYZER" "$RESULT_ROOT" || true
|
|
trap - EXIT INT TERM
|
|
log "PROFILE COMPLETE: $RESULT_ROOT"
|