feat: add EAGLE speculative decoding experiment (dsv4_pro6000_sglang_tp16_eagle)
This commit is contained in:
Zhiyi Hong
parent
4914ff4041
commit
c839230c0b
@ -0,0 +1,67 @@
|
|||||||
|
# ============================================================
|
||||||
|
# DeepSeek-V4-Pro SGLang TP=16 EP=2 + EAGLE speculative decoding
|
||||||
|
# Nodes: 10.101.0.11 (rank 0) + 10.101.0.13 (rank 1)
|
||||||
|
# GPUs: 16× RTX 6000D (85GB each)
|
||||||
|
#
|
||||||
|
# Key flags:
|
||||||
|
# SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2 → TP=16 with FP8 block=128
|
||||||
|
# --speculative-algorithm EAGLE → Eagle speculative decoding
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
EXPERIMENT="dsv4_pro6000_sglang_tp16_eagle"
|
||||||
|
MODEL_NAME="DeepSeek-V4-Pro"
|
||||||
|
MODEL_PATH="/data/hf_models/DeepSeek-V4-Pro"
|
||||||
|
SERVED_MODEL_NAME="default"
|
||||||
|
|
||||||
|
# ---- Nodes (IPs for reliable SSH) ----
|
||||||
|
HEAD_NODE="10.101.0.11"
|
||||||
|
WORKER_NODE="10.101.0.13"
|
||||||
|
HEAD_IP="10.101.0.11"
|
||||||
|
DIST_INIT_PORT="${DIST_INIT_PORT:-20000}"
|
||||||
|
SGLANG_PORT="${SGLANG_PORT:-30000}"
|
||||||
|
|
||||||
|
# ---- Parallelism ----
|
||||||
|
TP_SIZE="${TP_SIZE:-16}"
|
||||||
|
EP_SIZE="${EP_SIZE:-2}"
|
||||||
|
NNODES="${NNODES:-2}"
|
||||||
|
|
||||||
|
# ---- Docker ----
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
|
||||||
|
# ---- NCCL ----
|
||||||
|
NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}"
|
||||||
|
NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
|
||||||
|
|
||||||
|
# ---- SGLang memory / perf ----
|
||||||
|
# max_running_requests must >= max concurrency tested (128) to avoid queueing
|
||||||
|
# cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf
|
||||||
|
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
|
||||||
|
CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-64}"
|
||||||
|
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}"
|
||||||
|
|
||||||
|
# ---- EAGLE speculative decoding ----
|
||||||
|
SPECULATIVE_ALGORITHM="${SPECULATIVE_ALGORITHM:-EAGLE}"
|
||||||
|
SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}"
|
||||||
|
SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}"
|
||||||
|
SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}"
|
||||||
|
|
||||||
|
# ---- Benchmark ----
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||||
|
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||||||
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||||
|
WARMUP_REQUESTS="${WARMUP_REQUESTS:-16}"
|
||||||
|
NUM_REPETITIONS="${NUM_REPETITIONS:-3}"
|
||||||
|
|
||||||
|
# ---- Timeouts ----
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}"
|
||||||
|
HEALTH_CHECK_INTERVAL_S="${HEALTH_CHECK_INTERVAL_S:-5}"
|
||||||
|
|
||||||
|
# ---- Directories ----
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/results}"
|
||||||
|
|
||||||
|
# ---- Dry-run ----
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
229
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh
Executable file
229
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh
Executable file
@ -0,0 +1,229 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ================================================================
|
||||||
|
# ~20h benchmark matching dsv4_pro6000_vllm_tp_dp_matrix matrix.json
|
||||||
|
# Per-ISL TTFT SLO: 7s(min) -> 40s(max), stops concurrency sweep on violation
|
||||||
|
# OOM fault tolerance: auto-detect, skip combo, auto-restart server
|
||||||
|
# 3 repetitions for statistical robustness
|
||||||
|
# ================================================================
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
RUN_ID="${RUN_ID:-$(date +%Y%m%d_%H%M%S)}"
|
||||||
|
RESULT_DIR="${RESULT_BASE}/${RUN_ID}"
|
||||||
|
SUMMARY_FILE="${RESULT_DIR}/summary.csv"
|
||||||
|
mkdir -p "$RESULT_DIR"
|
||||||
|
echo "isl,osl,concurrency,tps,ttft_ms,tpot_ms,e2e_ms,slo_ms,status,elapsed_s" >> "$SUMMARY_FILE"
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# TTFT SLO per ISL (DeepSeek-V4-Pro, TP=16 cross-node, min=7000ms)
|
||||||
|
# ================================================================
|
||||||
|
get_slo_ms() {
|
||||||
|
local isl="$1"
|
||||||
|
if (( isl <= 2048 )); then echo 7000
|
||||||
|
elif (( isl <= 8192 )); then echo 10000
|
||||||
|
elif (( isl <= 32768 )); then echo 15000
|
||||||
|
elif (( isl <= 131072 )); then echo 25000
|
||||||
|
else echo 40000
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# Reference matrix from dsv4_pro6000_vllm_tp_dp_matrix/matrix.json
|
||||||
|
# ================================================================
|
||||||
|
declare -A MATRIX
|
||||||
|
MATRIX["1024,128"]=Y; MATRIX["1024,256"]=Y; MATRIX["1024,512"]=Y
|
||||||
|
MATRIX["1024,1024"]=Y; MATRIX["1024,2048"]=Y; MATRIX["1024,4096"]=Y
|
||||||
|
MATRIX["4096,128"]=Y; MATRIX["4096,256"]=Y; MATRIX["4096,512"]=Y
|
||||||
|
MATRIX["4096,1024"]=Y; MATRIX["4096,2048"]=Y; MATRIX["4096,4096"]=Y
|
||||||
|
MATRIX["8192,128"]=Y; MATRIX["8192,256"]=Y; MATRIX["8192,512"]=Y
|
||||||
|
MATRIX["8192,1024"]=Y; MATRIX["8192,2048"]=Y; MATRIX["8192,4096"]=Y
|
||||||
|
MATRIX["16384,128"]=Y; MATRIX["16384,256"]=Y; MATRIX["16384,512"]=Y
|
||||||
|
MATRIX["16384,1024"]=Y; MATRIX["16384,2048"]=Y; MATRIX["16384,4096"]=P
|
||||||
|
MATRIX["32768,128"]=Y; MATRIX["32768,256"]=Y; MATRIX["32768,512"]=Y
|
||||||
|
MATRIX["32768,1024"]=Y; MATRIX["32768,2048"]=Y; MATRIX["32768,4096"]=P
|
||||||
|
MATRIX["65536,128"]=Y; MATRIX["65536,256"]=Y; MATRIX["65536,512"]=Y
|
||||||
|
MATRIX["65536,1024"]=Y; MATRIX["65536,2048"]=P; MATRIX["65536,4096"]=N
|
||||||
|
MATRIX["131072,128"]=Y; MATRIX["131072,256"]=Y; MATRIX["131072,512"]=Y
|
||||||
|
MATRIX["131072,1024"]=P; MATRIX["131072,2048"]=N; MATRIX["131072,4096"]=N
|
||||||
|
MATRIX["262144,128"]=Y; MATRIX["262144,256"]=Y; MATRIX["262144,512"]=P
|
||||||
|
MATRIX["262144,1024"]=N; MATRIX["262144,2048"]=N; MATRIX["262144,4096"]=N
|
||||||
|
MATRIX["524288,128"]=Y; MATRIX["524288,256"]=P; MATRIX["524288,512"]=N
|
||||||
|
MATRIX["524288,1024"]=N; MATRIX["524288,2048"]=N; MATRIX["524288,4096"]=N
|
||||||
|
MATRIX["1048576,128"]=Y; MATRIX["1048576,256"]=P; MATRIX["1048576,512"]=N
|
||||||
|
MATRIX["1048576,1024"]=N; MATRIX["1048576,2048"]=N; MATRIX["1048576,4096"]=N
|
||||||
|
|
||||||
|
ISL_LIST=(1024 4096 8192 16384 32768 65536 131072 262144 524288 1048576)
|
||||||
|
OSL_LIST=(128 256 512 1024 2048 4096)
|
||||||
|
|
||||||
|
get_conc_for() {
|
||||||
|
local isl="$1"
|
||||||
|
if (( isl <= 4096 )); then echo "1 2 4 8 16 32 64 128"
|
||||||
|
elif (( isl <= 16384 )); then echo "1 2 4 8 16 32"
|
||||||
|
elif (( isl <= 65536 )); then echo "1 2 4 8"
|
||||||
|
else echo "1 2 4"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# Benchmark runner with per-ISL SLO + OOM tolerance
|
||||||
|
# ================================================================
|
||||||
|
run_one() {
|
||||||
|
local isl="$1" osl="$2" conc="$3"
|
||||||
|
local slo_ms; slo_ms=$(get_slo_ms "$isl")
|
||||||
|
local num_prompts=$(( conc * 5 ))
|
||||||
|
[[ $num_prompts -lt 10 ]] && num_prompts=10
|
||||||
|
local out="${RESULT_DIR}/isl${isl}_osl${osl}_c${conc}.json"
|
||||||
|
local logf="${RESULT_DIR}/isl${isl}_osl${osl}_c${conc}.log"
|
||||||
|
local start_ts=$(date +%s)
|
||||||
|
|
||||||
|
docker run --rm --network host \
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||||
|
-v "${DATASET_PATH}:${DATASET_PATH}:ro" \
|
||||||
|
-v "${RESULT_DIR}:${RESULT_DIR}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
--entrypoint python3 "$DOCKER_CLIENT_IMAGE" \
|
||||||
|
-m "${SGLANG_BENCH_MODULE}" \
|
||||||
|
--backend sglang --host "${HEAD_IP}" --port "${SGLANG_PORT}" \
|
||||||
|
--dataset-name "${BENCH_DATASET_NAME}" --dataset-path "${DATASET_PATH}" \
|
||||||
|
--random-input-len "$isl" --random-output-len "$osl" \
|
||||||
|
--random-range-ratio "${RANDOM_RANGE_RATIO}" \
|
||||||
|
--num-prompts "$num_prompts" --max-concurrency "$conc" \
|
||||||
|
--request-rate 10000 --warmup-requests "${WARMUP_REQUESTS}" \
|
||||||
|
--output-file "$out" --output-details --disable-tqdm \
|
||||||
|
> "$logf" 2>&1
|
||||||
|
local rc=$?
|
||||||
|
local elapsed=$(($(date +%s) - start_ts))
|
||||||
|
|
||||||
|
# OOM detection
|
||||||
|
if [[ $rc -ne 0 ]]; then
|
||||||
|
if grep -qi "out of memory\|CUDA error\|RESOURCE_EXHAUSTED" "$logf" 2>/dev/null; then
|
||||||
|
echo "${isl},${osl},${conc},-1,-1,-1,-1,${slo_ms},OOM,${elapsed}" >> "$SUMMARY_FILE"
|
||||||
|
log " OOM isl=${isl} osl=${osl} c=${conc}"; return 2
|
||||||
|
fi
|
||||||
|
echo "${isl},${osl},${conc},-1,-1,-1,-1,${slo_ms},FAIL,${elapsed}" >> "$SUMMARY_FILE"
|
||||||
|
log " FAIL isl=${isl} osl=${osl} c=${conc} rc=${rc}"; return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract metrics (JSONL: first line has aggregate summary)
|
||||||
|
local metrics=$(python3 -c "
|
||||||
|
import json
|
||||||
|
for line in open('${out}'):
|
||||||
|
line=line.strip()
|
||||||
|
if not line: continue
|
||||||
|
try:
|
||||||
|
d=json.loads(line)
|
||||||
|
tps=d.get('total_throughput',d.get('tps',-1))
|
||||||
|
ttft=d.get('mean_ttft_ms',d.get('ttft_mean',-1))
|
||||||
|
tpot=d.get('mean_tpot_ms',d.get('tpot_mean',-1))
|
||||||
|
e2e=d.get('mean_e2e_latency_ms',d.get('e2e_latency_mean',-1))
|
||||||
|
if tps!=-1 or ttft!=-1:
|
||||||
|
print(f'{round(tps,1)},{round(ttft,1)},{round(tpot,1)},{round(e2e,1)}')
|
||||||
|
break
|
||||||
|
except: pass
|
||||||
|
" 2>/dev/null || echo "-1,-1,-1,-1")
|
||||||
|
local tps=$(echo "$metrics" | cut -d, -f1)
|
||||||
|
local ttft=$(echo "$metrics" | cut -d, -f2)
|
||||||
|
local tpot=$(echo "$metrics" | cut -d, -f3)
|
||||||
|
local e2e=$(echo "$metrics" | cut -d, -f4)
|
||||||
|
|
||||||
|
echo "${isl},${osl},${conc},${tps},${ttft},${tpot},${e2e},${slo_ms},OK,${elapsed}" >> "$SUMMARY_FILE"
|
||||||
|
|
||||||
|
# TTFT SLO check (per-ISL)
|
||||||
|
if [[ "$ttft" != "-1" ]]; then
|
||||||
|
if (( $(echo "$ttft > ${slo_ms}" | bc -l 2>/dev/null) )); then
|
||||||
|
log " SLO! ttft=${ttft}ms > slo=${slo_ms}ms -> stop higher conc"
|
||||||
|
return 3
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
log " OK isl=${isl} osl=${osl} c=${conc} tps=${tps} ttft=${ttft}ms slo=${slo_ms}ms"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# OOM recovery: restart server if it died
|
||||||
|
# ================================================================
|
||||||
|
check_and_recover_server() {
|
||||||
|
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
log "WARN: Server health failed. Checking..."
|
||||||
|
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
|
||||||
|
"docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
|
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
|
||||||
|
"docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
|
if [[ -z "$hok" || -z "$wok" ]]; then
|
||||||
|
log "ERROR: Containers died. Restarting server..."
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }
|
||||||
|
log "Server restarted."
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ================================================================
|
||||||
|
# Main
|
||||||
|
# ================================================================
|
||||||
|
main() {
|
||||||
|
log "=== ${EXPERIMENT} Benchmark ==="
|
||||||
|
log "Matrix: dsv4_pro6000_vllm_tp_dp_matrix"
|
||||||
|
log "TTFT SLO: per-ISL (7s-40s), Reps: ${NUM_REPETITIONS}"
|
||||||
|
log "Server: ${HEAD_IP}:${SGLANG_PORT}, Results: ${RESULT_DIR}"
|
||||||
|
|
||||||
|
if ! curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
log "FATAL: Server not healthy"; exit 1
|
||||||
|
fi
|
||||||
|
log "Server health OK."
|
||||||
|
|
||||||
|
local total=0 ok=0 fail=0 oom=0 slo=0 skip=0
|
||||||
|
local start_time=$(date +%s)
|
||||||
|
|
||||||
|
for rep in $(seq 1 ${NUM_REPETITIONS}); do
|
||||||
|
log "========== REPETITION ${rep}/${NUM_REPETITIONS} =========="
|
||||||
|
|
||||||
|
for isl in "${ISL_LIST[@]}"; do
|
||||||
|
if ! check_and_recover_server; then exit 1; fi
|
||||||
|
for osl in "${OSL_LIST[@]}"; do
|
||||||
|
local tag="${MATRIX[${isl},${osl}]:-N}"
|
||||||
|
[[ "$tag" == "N" ]] && { ((skip++)); continue; }
|
||||||
|
log "--- isl=${isl} osl=${osl} [${tag}] ---"
|
||||||
|
|
||||||
|
local conc_list=($(get_conc_for "$isl"))
|
||||||
|
local combo_oom=0 combo_slo=0
|
||||||
|
|
||||||
|
for conc in "${conc_list[@]}"; do
|
||||||
|
[[ $combo_oom -eq 1 ]] && { ((skip++)); continue; }
|
||||||
|
[[ $combo_slo -eq 1 ]] && { ((slo++)); continue; }
|
||||||
|
((total++))
|
||||||
|
run_one "$isl" "$osl" "$conc"
|
||||||
|
case $? in
|
||||||
|
0) ((ok++)) ;;
|
||||||
|
1) ((fail++)) ;;
|
||||||
|
2) ((oom++)); combo_oom=1 ;;
|
||||||
|
3) ((slo++)); combo_slo=1 ;;
|
||||||
|
esac
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
|
||||||
|
local elapsed=$(($(date +%s) - start_time))
|
||||||
|
local eta="N/A"
|
||||||
|
if (( ok > 0 )); then
|
||||||
|
local rate=$(echo "scale=3; $ok/$elapsed" | bc 2>/dev/null || echo "0")
|
||||||
|
if [[ "$rate" != "0" ]]; then
|
||||||
|
local eta_s=$(echo "scale=0; ($total-$ok)/$rate" | bc 2>/dev/null || echo "0")
|
||||||
|
eta="$(date -u -d @${eta_s} +%Hh%Mm 2>/dev/null || echo '?')"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
log "PROGRESS ok=${ok}/${total} fail=${fail} oom=${oom} slo=${slo} elapsed=$(echo "scale=1;$elapsed/3600"|bc)h ETA=${eta}"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
local total_h=$(echo "scale=1; $(($(date +%s)-start_time))/3600" | bc)
|
||||||
|
log "=== COMPLETE: total=${total} ok=${ok} fail=${fail} oom=${oom} slo=${slo} skip=${skip} time=${total_h}h ==="
|
||||||
|
log "Results: ${RESULT_DIR}"
|
||||||
|
}
|
||||||
|
|
||||||
|
main
|
||||||
9
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_and_bench.sh
Executable file
9
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_and_bench.sh
Executable file
@ -0,0 +1,9 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||||
|
log "=== Step 1: Starting SGLang multi-node server (EAGLE) ==="
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_multinode.sh"
|
||||||
|
log "=== Step 2: Starting benchmark ==="
|
||||||
|
bash "${SCRIPT_DIR}/run_batch.sh"
|
||||||
|
log "=== All done! ==="
|
||||||
82
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh
Executable file
82
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh
Executable file
@ -0,0 +1,82 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ============================================================
|
||||||
|
# Start SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding
|
||||||
|
# Usage: ./start_sglang_multinode.sh [--dry-run]
|
||||||
|
# ============================================================
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
HEAD_CONTAINER="${EXPERIMENT}_head"
|
||||||
|
WORKER_CONTAINER="${EXPERIMENT}_worker"
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||||
|
|
||||||
|
build_docker_cmd() {
|
||||||
|
local node_rank="$1" container_name="$2"
|
||||||
|
cat <<DOCKER_CMD
|
||||||
|
docker rm -f ${container_name} 2>/dev/null;
|
||||||
|
docker run -d --name ${container_name} \\
|
||||||
|
--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\
|
||||||
|
-v ${MODEL_PATH}:${MODEL_PATH}:ro \\
|
||||||
|
-e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\
|
||||||
|
-e NCCL_DEBUG=${NCCL_DEBUG} \\
|
||||||
|
-e SGLANG_SHARED_EXPERT_TP1=1 \\
|
||||||
|
${DOCKER_IMAGE} \\
|
||||||
|
python3 -m sglang.launch_server \\
|
||||||
|
--model-path ${MODEL_PATH} \\
|
||||||
|
--tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\
|
||||||
|
--node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\
|
||||||
|
--trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\
|
||||||
|
--mem-fraction-static ${MEM_FRACTION_STATIC} \\
|
||||||
|
--cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\
|
||||||
|
--max-running-requests ${MAX_RUNNING_REQUESTS} \\
|
||||||
|
--speculative-algorithm ${SPECULATIVE_ALGORITHM} \\
|
||||||
|
--speculative-num-steps ${SPECULATIVE_NUM_STEPS} \\
|
||||||
|
--speculative-eagle-topk ${SPECULATIVE_EAGLE_TOPK} \\
|
||||||
|
--speculative-num-draft-tokens ${SPECULATIVE_NUM_DRAFT_TOKENS}
|
||||||
|
DOCKER_CMD
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
log "=== SGLang TP=${TP_SIZE} EP=${EP_SIZE} + EAGLE ==="
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "0" ]]; then
|
||||||
|
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true
|
||||||
|
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true
|
||||||
|
sleep 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..."
|
||||||
|
local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")"
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
||||||
|
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null
|
||||||
|
log "Worker started."
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..."
|
||||||
|
cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")"
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
||||||
|
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null
|
||||||
|
log "Head started."
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi
|
||||||
|
log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..."
|
||||||
|
for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do
|
||||||
|
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
log "Server healthy! (${i}s)"; return 0
|
||||||
|
fi
|
||||||
|
local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
||||||
|
local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
||||||
|
if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi
|
||||||
|
if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi
|
||||||
|
sleep "${HEALTH_CHECK_INTERVAL_S}"
|
||||||
|
done
|
||||||
|
log "ERROR: Timeout"; return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
||||||
|
log "Stopping ${EXPERIMENT} ..."
|
||||||
|
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'"
|
||||||
|
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'"
|
||||||
|
log "Done."
|
||||||
Loading…
x
Reference in New Issue
Block a user