qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pro6000): 部署/测试解耦 - deploy 层支持多节点与 vLLM，新增 6 个 profile

Browse Source 
- sskj.deploy runtime 支持 NODE_HOSTS 多节点编排(ssh 分发/本地 rank/LOCAL_NODE_RANK)
  与 ENGINE=vllm 启动(SERVER_CMD)，容器名按 rank 自动唯一
- scripts/common/deploy_cli.sh 新增 deploy_stop/status/multinode helper 与 node-rank 透传
- src/sskj/common/env.py 修复嵌套 ${VAR:-${OTHER}/path} 展开(平衡花括号扫描)
- deploy/profiles/pro6000/ 新增 6 个 profile: tp16/tp16_eagle/glm52(多节点)、
  sglang/vllm tp_dp_matrix、qwen3(单节点)
- 6 个实验 start/stop 脚本改为 deploy 薄包装，run_bench/adaptive 的 server 启停走
  deploy_render_args/deploy_start/deploy_stop，tp16 新增 matrix.json
- 首次入库 glm52_pro6000_sglang_multinode_tp16 实验目录；ops/README.md 补 pro6000 章节
- 实测通过: 单节点 dsv4 sglang/vllm 链路 + tp16 双节点启动/bench/清理
This commit is contained in:
Zhiyi Hong 2026-08-03 15:17:41 +08:00
parent 3761d75b00
commit 9acf9fdfdb
81 changed files with 17421 additions and 705 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@ -76,14 +76,21 @@ build/
sglang_sm120_cache/
vllm_sm120_cache/
*_sm120_cache/
sglang_nightly_cu13_cache/
# 一次性 bench 输出
bench-output/
# 单数 dataset/（可重新下载，与 datasets/ 同源）
dataset/
# 逐请求原始日志（体积大；汇总见 results.json / report.md）
experiments/**/raw_outputs/
**/dummy_sharegpt.json
# pro6000 统一 bench CLI 的一次性测试输出（ops_ 前缀 run_id）
experiments/pro6000/*/results/ops_*/
# Virtual environments (keep only docs under envs/)
envs/*
!envs/README.md
@ -96,3 +103,6 @@ skills-lock.json
# 临时备份文件
*.bak.glm52orig
*.bak
*.bak.*
*.tmp

34
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16.env Normal file
View File

@ -0,0 +1,34 @@
# DeepSeek-V4-Pro SGLang multi-node TP=16 EP=2 deployment profile (2x RTX 6000D).
# Nodes: 10.101.0.11 (rank 0) + 10.101.0.13 (rank 1), 16x RTX 6000D (85GB).
# Key: SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2 -> TP=16 works with FP8 block=128.
# Model-team only. Ops only run `python -m sskj.bench` against the served URL.
PLATFORM=pro6000
EXPERIMENT=dsv4_pro6000_sglang_tp16
MODEL_NAME=DeepSeek-V4-Pro
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
MODEL_PATH=/data/hf_models/DeepSeek-V4-Pro
SERVED_MODEL_NAME=default
PORT=30000
HEALTH_PATH=/health
HEALTH_HOST=10.101.0.11
HEALTH_WAIT_S=2400
CONTAINER_PYTHON=python3
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
NNODES=2
NODE_HOSTS="10.101.0.11 10.101.0.13"
NODE_SSH_USER=root
MASTER_IP=10.101.0.11
DIST_PORT=20000
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="NCCL_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN SGLANG_SHARED_EXPERT_TP1=1 PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
LAUNCH_ARGS="--model-path ${MODEL_PATH} --tp-size ${TP} --ep-size 2 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --host 0.0.0.0 --port ${PORT} --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256"

32
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16_eagle.env Normal file
View File

@ -0,0 +1,32 @@
# DeepSeek-V4-Pro SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding
# (2x RTX 6000D). Same topology as dsv4_pro6000_sglang_tp16, plus EAGLE flags.
PLATFORM=pro6000
EXPERIMENT=dsv4_pro6000_sglang_tp16_eagle
MODEL_NAME=DeepSeek-V4-Pro
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
MODEL_PATH=/data/hf_models/DeepSeek-V4-Pro
SERVED_MODEL_NAME=default
PORT=30000
HEALTH_PATH=/health
HEALTH_HOST=10.101.0.11
HEALTH_WAIT_S=2400
CONTAINER_PYTHON=python3
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
NNODES=2
NODE_HOSTS="10.101.0.11 10.101.0.13"
NODE_SSH_USER=root
MASTER_IP=10.101.0.11
DIST_PORT=20000
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="NCCL_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN SGLANG_SHARED_EXPERT_TP1=1 PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
LAUNCH_ARGS="--model-path ${MODEL_PATH} --tp-size ${TP} --ep-size 2 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --host 0.0.0.0 --port ${PORT} --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4"

24
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp_dp_matrix.env Normal file
View File

@ -0,0 +1,24 @@
# DeepSeek-V4-Flash SGLang TPxDP matrix profile on a single RTX 6000D node (8 GPUs).
# TP=2/DP=4 is excluded because Marlin weight loading OOMs on this machine.
PLATFORM=pro6000
EXPERIMENT=dsv4_pro6000_sglang_tp_dp_matrix
MODEL_NAME=DeepSeek-V4-Flash
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
CONTAINER_NAME=${EXPERIMENT}_sglang_tp${TP}_dp${DP}
MODEL_PATH=/data/6000D/DeepSeek-V4-Flash
SERVED_MODEL_NAME=deepseek-v4-flash
PORT=30031
HEALTH_PATH=/health
HEALTH_WAIT_S=2400
CONTAINER_PYTHON=python3
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True"
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --moe-runner-backend auto --mem-fraction-static 0.9 --context-length 131072 --max-running-requests 64 --host 0.0.0.0 --port ${PORT}"

24
deploy/profiles/pro6000/dsv4_pro6000_vllm_tp_dp_matrix.env Normal file
View File

@ -0,0 +1,24 @@
# DeepSeek-V4-Flash vLLM TPxDP matrix profile on a single RTX 6000D node (8 GPUs).
PLATFORM=pro6000
EXPERIMENT=dsv4_pro6000_vllm_tp_dp_matrix
MODEL_NAME=DeepSeek-V4-Flash
ENGINE=vllm
SERVER_CMD=vllm serve
RUNTIME=docker
DOCKER_IMAGE=vllm-sm120-dsv4:0.25.1-fi0.6.14
CONTAINER_NAME=${EXPERIMENT}_vllm_tp${TP}_dp${DP}
MODEL_PATH=/data/6000D/DeepSeek-V4-Flash
SERVED_MODEL_NAME=deepseek-v4-flash
PORT=30030
HEALTH_PATH=/health
HEALTH_WAIT_S=2400
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
LAUNCH_ARGS="--model ${MODEL_PATH} --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size ${TP} --gpu-memory-utilization 0.9 --max-model-len 131072 --max-num-seqs 128 --host 0.0.0.0 --port ${PORT}"
DP_FLAG="--data-parallel-size ${DP}"

37
deploy/profiles/pro6000/glm52_pro6000_sglang_tp16.env Normal file
View File

@ -0,0 +1,37 @@
# GLM-5.2-FP8 SGLang multi-node TP=16 deployment profile (2x RTX 6000D).
# node0 = 174.1.51.5 (pro6000D.1, HTTP API), node1 = 174.1.51.7 (pro6000D.3,
# compute-only). Model ~700GB fp8 so 2-node TP=16 is mandatory.
# Requires the tilelang/dsa patch mounts below (experiment-dir sources).
PLATFORM=pro6000
EXPERIMENT=glm52_pro6000_sglang_multinode_tp16
MODEL_NAME=GLM-5.2-FP8
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
CONTAINER_NAME=${EXPERIMENT}_sglang_tp${TP}_dp${DP}_node${NODE_RANK}
MODEL_PATH=/data/hf_models/GLM-5.2-FP8
SERVED_MODEL_NAME=GLM-5.2-FP8
PORT=30031
HEALTH_PATH=/health
HEALTH_HOST=174.1.51.5
HEALTH_WAIT_S=2400
CONTAINER_PYTHON=python3
TP=16
DP=1
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
NNODES=2
NODE_HOSTS="174.1.51.5 174.1.51.7"
NODE_SSH_USER=root
MASTER_IP=174.1.51.5
DIST_PORT=50000
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:256 NCCL_IB_HCA=mlx5_0,mlx5_3 NCCL_MIN_NCHANNELS=8 NCCL_IB_QPS_PER_CONNECTION=4 NCCL_NET_GDR_LEVEL=PHB NCCL_SOCKET_IFNAME=eth1 NCCL_IB_RETRY_CNT=14 NCCL_IB_TIMEOUT=16"
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro ${ROOT_DIR}/sglang_nightly_cu13_cache:/root/.cache"
PATCH_MOUNTS="${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/tilelang_kernel_sm120.py:/sgl-workspace/sglang/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_backend.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa_backend.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_indexer.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa/dsa_indexer.py:ro"
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --dp-size ${DP} --quantization fp8 --kv-cache-dtype bfloat16 --moe-runner-backend auto --mem-fraction-static 0.80 --context-length 131072 --max-running-requests 64 --cuda-graph-backend-decode disabled --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --host 0.0.0.0 --port ${PORT} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --nnodes ${NNODES} --node-rank ${NODE_RANK}"

26
deploy/profiles/pro6000/qwen3_235b_pro6000_sglang_tp8.env Normal file
View File

@ -0,0 +1,26 @@
# Qwen3-235B-A22B SGLang TP=8 profile on a single RTX 6000D node (8 GPUs).
# NVIDIA counterpart of p800/qwen3_235b_p800_sglang_tp8 (same workload for
# comparability: isl=2048 osl=2048 c=16).
PLATFORM=pro6000
EXPERIMENT=qwen3_235b_pro6000_sglang_tp8
MODEL_NAME=Qwen3-235B-A22B
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1
CONTAINER_NAME=qwen3_235b_pro6000_sglang_tp8
MODEL_PATH=/data/6000D/Qwen3-235B-A22B
SERVED_MODEL_NAME=/data/6000D/Qwen3-235B-A22B
PORT=30010
HEALTH_PATH=/health
HEALTH_WAIT_S=1800
CONTAINER_PYTHON=python3
TP=8
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True"
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
VOLUMES="/data:/data"
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --mem-fraction-static 0.9 --context-length 8192 --max-running-requests 16 --host 0.0.0.0 --port ${PORT}"

128
experiments/pro6000/dsv4_pro6000_sglang_tp16/matrix.json Normal file
View File

@ -0,0 +1,128 @@
{
"comment": "ISL/OSL matrix for dsv4_pro6000_sglang_tp16 (multi-node TP16 EP2). Y=must test, P=optional, N=skip. Extracted from run_batch.sh legacy inline matrix; concurrency per ISL mirrors get_conc_for().",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"4096": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "P",
"4096": "N"
},
"131072": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "P",
"2048": "N",
"4096": "N"
},
"262144": {
"128": "Y",
"256": "Y",
"512": "P",
"1024": "N",
"2048": "N",
"4096": "N"
},
"524288": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
},
"1048576": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
}
},
"concurrency": {
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 128
},
"8192": {
"low": 1,
"high": 32
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 8
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 4
},
"524288": {
"low": 1,
"high": 4
},
"1048576": {
"low": 1,
"high": 4
}
}
}

4
experiments/pro6000/dsv4_pro6000_sglang_tp16/run_batch.sh
View File

@ -152,9 +152,9 @@ check_and_recover_server() {
fi
log "WARN: Server health failed. Checking..."
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
"docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "")
"docker ps --filter name=${EXPERIMENT}_node0 --format '{{.Names}}' 2>/dev/null" || echo "")
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
"docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "")
"docker ps --filter name=${EXPERIMENT}_node1 --format '{{.Names}}' 2>/dev/null" || echo "")
if [[ -z "$hok" || -z "$wok" ]]; then
log "ERROR: Containers died. Restarting server..."
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }

106
experiments/pro6000/dsv4_pro6000_sglang_tp16/start_sglang_multinode.sh
View File

@ -1,84 +1,40 @@
#!/usr/bin/env bash
# ============================================================
# Start SGLang multi-node TP=16 EP=2 across 2 nodes.
# Key: SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2
# Usage: ./start_sglang_multinode.sh [--dry-run]
# ============================================================
# Start SGLang multi-node TP=16 EP=2 across 2 nodes through the shared
# deployment layer (`python -m sskj.deploy start`, NODE_HOSTS orchestration).
# Usage: start_sglang_multinode.sh [--dry-run]
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs"
DRY_RUN="${DRY_RUN:-0}"
HEAD_CONTAINER="${EXPERIMENT}_head"
WORKER_CONTAINER="${EXPERIMENT}_worker"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16}"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} via deploy profile ${DEPLOY_PROFILE} ==="
build_docker_cmd() {
local node_rank="$1" container_name="$2"
cat <<DOCKER_CMD
docker rm -f ${container_name} 2>/dev/null;
docker run -d --name ${container_name} \\
--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\
-v ${MODEL_PATH}:${MODEL_PATH}:ro \\
-e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\
-e NCCL_DEBUG=${NCCL_DEBUG} \\
-e SGLANG_SHARED_EXPERT_TP1=1 \\
${DOCKER_IMAGE} \\
python3 -m sglang.launch_server \\
--model-path ${MODEL_PATH} \\
--tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\
--node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\
--trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\
--mem-fraction-static ${MEM_FRACTION_STATIC} \\
--cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\
--max-running-requests ${MAX_RUNNING_REQUESTS}
DOCKER_CMD
}
main() {
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} ==="
if [[ "$DRY_RUN" == "0" ]]; then
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true
sleep 2
fi
log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..."
local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")"
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null
log "Worker started."
fi
sleep 5
log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..."
cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")"
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null
log "Head started."
fi
# Health check
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi
log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..."
for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
log "Server healthy! (${i}s)"
return 0
fi
local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" \
"docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" \
"docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi
if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi
if (( i % 12 == 0 )); then log " waiting... ${i}s"; fi
sleep "${HEALTH_CHECK_INTERVAL_S}"
done
log "ERROR: Timeout"; return 1
}
main "$@"
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP_SIZE" --dp "1" \
--port "${SGLANG_PORT:-30000}" \
--model-path "$MODEL_PATH" \
--log-dir "${RUNTIME_BASE}/logs" \
--dry-run
else
deploy_start_multinode \
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
"${RUNTIME_BASE}/logs" \
"${SGLANG_PORT:-30000}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi

38
experiments/pro6000/dsv4_pro6000_sglang_tp16/stop_sglang_multinode.sh
View File

@ -1,14 +1,36 @@
#!/usr/bin/env bash
# Stop SGLang multi-node server and clean up.
set -euo pipefail
# Stop SGLang multi-node server (both nodes) through the shared deployment layer.
# Usage: stop_sglang_multinode.sh [--dry-run]
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
log() { echo "[$(date '+%H:%M:%S')] $*"; }
DRY_RUN="${DRY_RUN:-0}"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16}"
log "Stopping ${EXPERIMENT} ..."
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" \
"docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'"
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" \
"docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'"
log "=== Stopping SGLang multi-node via deploy profile ${DEPLOY_PROFILE} ==="
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP_SIZE" --dp "1" \
--port "${SGLANG_PORT:-30000}" \
--model-path "$MODEL_PATH" \
--container-name "$EXPERIMENT" \
--dry-run
else
deploy_stop \
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
"${SGLANG_PORT:-30000}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi
log "Done."

4
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/run_batch.sh
View File

@ -152,9 +152,9 @@ check_and_recover_server() {
fi
log "WARN: Server health failed. Checking..."
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
"docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "")
"docker ps --filter name=${EXPERIMENT}_node0 --format '{{.Names}}' 2>/dev/null" || echo "")
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
"docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "")
"docker ps --filter name=${EXPERIMENT}_node1 --format '{{.Names}}' 2>/dev/null" || echo "")
if [[ -z "$hok" || -z "$wok" ]]; then
log "ERROR: Containers died. Restarting server..."
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }

104
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/start_sglang_multinode.sh
View File

@ -1,82 +1,40 @@
#!/usr/bin/env bash
# ============================================================
# Start SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding
# Usage: ./start_sglang_multinode.sh [--dry-run]
# ============================================================
# Start SGLang multi-node TP=16 EP=2 across 2 nodes through the shared
# deployment layer (`python -m sskj.deploy start`, NODE_HOSTS orchestration).
# Usage: start_sglang_multinode.sh [--dry-run]
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs"
DRY_RUN="${DRY_RUN:-0}"
HEAD_CONTAINER="${EXPERIMENT}_head"
WORKER_CONTAINER="${EXPERIMENT}_worker"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16_eagle}"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} via deploy profile ${DEPLOY_PROFILE} ==="
build_docker_cmd() {
local node_rank="$1" container_name="$2"
cat <<DOCKER_CMD
docker rm -f ${container_name} 2>/dev/null;
docker run -d --name ${container_name} \\
--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\
-v ${MODEL_PATH}:${MODEL_PATH}:ro \\
-e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\
-e NCCL_DEBUG=${NCCL_DEBUG} \\
-e SGLANG_SHARED_EXPERT_TP1=1 \\
${DOCKER_IMAGE} \\
python3 -m sglang.launch_server \\
--model-path ${MODEL_PATH} \\
--tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\
--node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\
--trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\
--mem-fraction-static ${MEM_FRACTION_STATIC} \\
--cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\
--max-running-requests ${MAX_RUNNING_REQUESTS} \\
--speculative-algorithm ${SPECULATIVE_ALGORITHM} \\
--speculative-num-steps ${SPECULATIVE_NUM_STEPS} \\
--speculative-eagle-topk ${SPECULATIVE_EAGLE_TOPK} \\
--speculative-num-draft-tokens ${SPECULATIVE_NUM_DRAFT_TOKENS}
DOCKER_CMD
}
main() {
log "=== SGLang TP=${TP_SIZE} EP=${EP_SIZE} + EAGLE ==="
if [[ "$DRY_RUN" == "0" ]]; then
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true
sleep 2
fi
log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..."
local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")"
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null
log "Worker started."
fi
sleep 5
log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..."
cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")"
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null
log "Head started."
fi
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi
log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..."
for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
log "Server healthy! (${i}s)"; return 0
fi
local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi
if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi
sleep "${HEALTH_CHECK_INTERVAL_S}"
done
log "ERROR: Timeout"; return 1
}
main "$@"
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP_SIZE" --dp "1" \
--port "${SGLANG_PORT:-30000}" \
--model-path "$MODEL_PATH" \
--log-dir "${RUNTIME_BASE}/logs" \
--dry-run
else
deploy_start_multinode \
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
"${RUNTIME_BASE}/logs" \
"${SGLANG_PORT:-30000}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi

37
experiments/pro6000/dsv4_pro6000_sglang_tp16_eagle/stop_sglang_multinode.sh
View File

@ -1,9 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
# Stop SGLang multi-node server (both nodes) through the shared deployment layer.
# Usage: stop_sglang_multinode.sh [--dry-run]
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
log() { echo "[$(date '+%H:%M:%S')] $*"; }
log "Stopping ${EXPERIMENT} ..."
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'"
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DRY_RUN="${DRY_RUN:-0}"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16_eagle}"
log "=== Stopping SGLang multi-node via deploy profile ${DEPLOY_PROFILE} ==="
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP_SIZE" --dp "1" \
--port "${SGLANG_PORT:-30000}" \
--model-path "$MODEL_PATH" \
--container-name "$EXPERIMENT" \
--dry-run
else
deploy_stop \
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
"${SGLANG_PORT:-30000}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi
log "Done."

44
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}"
ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT"
@ -37,29 +41,9 @@ engine_is_healthy() {
engine_stop_server() {
local tp="$1"
local dp="$2"
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
if [[ -n "${CONTAINER_NAME:-}" ]]; then
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
sleep 5
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
fi
elif kill -0 "$pid" 2>/dev/null; then
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
fi
if [[ -z "${CONTAINER_NAME:-}" ]]; then
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
fi
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
@ -67,21 +51,7 @@ engine_stop_server() {
engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--context-length "$CONTEXT_LENGTH"
--max-running-requests "$MAX_RUNNING_REQUESTS"
--host 0.0.0.0
--port "$ENGINE_PORT"
)
if (( dp > 1 )); then
args+=(--dp-size "$dp")
fi
printf '%q ' "${args[@]}"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {

48
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}"
ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT"
@ -37,29 +41,9 @@ engine_is_healthy() {
engine_stop_server() {
local tp="$1"
local dp="$2"
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
if [[ -n "${CONTAINER_NAME:-}" ]]; then
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
sleep 5
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
fi
elif kill -0 "$pid" 2>/dev/null; then
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
fi
if [[ -z "${CONTAINER_NAME:-}" ]]; then
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
fi
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
@ -67,21 +51,7 @@ engine_stop_server() {
engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--context-length "$CONTEXT_LENGTH"
--max-running-requests "$MAX_RUNNING_REQUESTS"
--host 0.0.0.0
--port "$ENGINE_PORT"
)
if (( dp > 1 )); then
args+=(--dp-size "$dp")
fi
printf '%q ' "${args[@]}"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {
@ -181,6 +151,8 @@ export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
@ -189,5 +161,3 @@ export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
# shapes in this TP/DP group. Zero disables this rule.
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
adaptive_main "$@"

53
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh
View File

@ -11,6 +11,8 @@ source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_BASE="${SCRIPT_DIR}/results"
@ -51,33 +53,8 @@ stop_server() {
local tp="$1"
local dp="$2"
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
# In container-reuse mode, pid is the server PID inside the container.
if [[ -n "${CONTAINER_NAME:-}" ]]; then
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
log "stopping sglang server inside container (tp=${tp}, dp=${dp}, pid=${pid})"
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
sleep 5
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
fi
else
if kill -0 "$pid" 2>/dev/null; then
log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
fi
rm -f "$pid_file"
fi
# Fallback: remove any Docker container started by this experiment (legacy mode).
if [[ -z "${CONTAINER_NAME:-}" ]]; then
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
fi
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
# Fallback: kill any SGLang launch processes for this model.
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
@ -89,23 +66,11 @@ build_server_args() {
local tp="$1"
local dp="$2"
local args=(
"python3 -m sglang.launch_server" --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--context-length "$CONTEXT_LENGTH"
--max-running-requests "$MAX_RUNNING_REQUESTS"
--host 0.0.0.0
--port "$SGLANG_PORT"
)
if [[ "$dp" -gt 1 ]]; then
args+=(
--dp-size "$dp"
)
fi
printf '%s ' "${args[@]}"
deploy_render_args \
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
"$tp" "$dp" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH"
}
start_server() {

118
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh
View File

@ -1,105 +1,33 @@
#!/usr/bin/env bash
# Start SGLang server in Docker for a given TPxDP configuration.
# Start the SGLang TPxDP server through the shared deployment layer.
# Usage: start_sglang_docker.sh <TP> <DP>
#
# Uses the verified SGLang SM120 image and keeps its JIT cache on
# persistent storage. The container is removed automatically on stop.
set -e
set -Eeuo pipefail
TP="${1}"
DP="${2}"
TP="${1:-}"
DP="${2:-}"
if [[ -z "$TP" || -z "$DP" ]]; then
echo "Usage: $0 <TP> <DP>"
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
PORT="${SGLANG_PORT:-30031}"
NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
rm -f "$PID_FILE"
# Clean up any stale container with the same name.
docker rm -f "$NAME" >/dev/null 2>&1 || true
SERVER_ARGS=(
-m sglang.launch_server
--model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$TP"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--context-length "$CONTEXT_LENGTH"
--max-running-requests "$MAX_RUNNING_REQUESTS"
--host 0.0.0.0
--port "$PORT"
)
if [[ "$DP" -gt 1 ]]; then
SERVER_ARGS+=(
--dp-size "$DP"
)
fi
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="
echo "Image: $IMAGE"
echo "Model: $MODEL_PATH"
echo "Container name: $NAME"
echo "Host port: $PORT"
echo "Command: python3 ${SERVER_ARGS_STR}"
echo "Log: $LOG"
# Run docker in the foreground so that killing the host process stops the
# container (the --rm flag ensures cleanup). nohup lets us background it and
# capture the host PID in the same way as the bare-metal start script.
nohup docker run --rm \
--name "$NAME" \
--gpus all \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint python3 \
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
-e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \
"$IMAGE" \
"${SERVER_ARGS[@]}" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
echo "Waiting for health on port ${PORT}..."
for i in $(seq 1 240); do
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
echo "SGLang server is ready at http://127.0.0.1:${PORT}"
echo "Log: $LOG"
exit 0
fi
if ! kill -0 $PID 2>/dev/null; then
echo "ERROR: Docker SGLang server exited early"
tail -200 "$LOG"
exit 1
fi
echo "Waiting... ($i/240)"
sleep 5
done
echo "ERROR: Docker SGLang server not healthy after 240 retries"
tail -200 "$LOG"
exit 1
log "starting SGLang server tp=${TP} dp=${DP} via deploy profile"
deploy_start \
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
"$TP" "$DP" \
"${RUNTIME_BASE}/logs" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH" \
"${EXPERIMENT}_sglang_tp${TP}_dp${DP}"

29
experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/stop_sglang_docker.sh Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Stop the SGLang TPxDP server through the shared deployment layer.
# Usage: stop_sglang_docker.sh <TP> <DP>
set -Eeuo pipefail
TP="${1:-}"
DP="${2:-}"
if [[ -z "$TP" || -z "$DP" ]]; then
echo "Usage: $0 <TP> <DP>"
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
log "stopping SGLang server tp=${TP} dp=${DP} via deploy profile"
deploy_stop \
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
"$TP" "$DP" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH" \
"${EXPERIMENT}_sglang_tp${TP}_dp${DP}"

36
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}"
ENGINE="vllm"
ENGINE_PORT="$VLLM_PORT"
@ -38,20 +42,9 @@ engine_is_healthy() {
engine_stop_server() {
local tp="$1"
local dp="$2"
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
fi
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
log "stopping vllm server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_vllm_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
@ -59,22 +52,7 @@ engine_stop_server() {
engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
vllm serve "$MODEL_PATH"
--trust-remote-code
--kv-cache-dtype "$KV_CACHE_DTYPE"
--block-size "$BLOCK_SIZE"
--tensor-parallel-size "$tp"
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--host 0.0.0.0
--port "$ENGINE_PORT"
)
if (( dp > 1 )); then
args+=(--data-parallel-size "$dp")
fi
printf '%q ' "${args[@]}"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {

40
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}"
ENGINE="vllm"
ENGINE_PORT="$VLLM_PORT"
@ -38,20 +42,9 @@ engine_is_healthy() {
engine_stop_server() {
local tp="$1"
local dp="$2"
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
fi
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
log "stopping vllm server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_vllm_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
@ -59,22 +52,7 @@ engine_stop_server() {
engine_build_server_args() {
local tp="$1"
local dp="$2"
local -a args=(
vllm serve "$MODEL_PATH"
--trust-remote-code
--kv-cache-dtype "$KV_CACHE_DTYPE"
--block-size "$BLOCK_SIZE"
--tensor-parallel-size "$tp"
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--host 0.0.0.0
--port "$ENGINE_PORT"
)
if (( dp > 1 )); then
args+=(--data-parallel-size "$dp")
fi
printf '%q ' "${args[@]}"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {
@ -163,6 +141,8 @@ export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
@ -171,5 +151,3 @@ export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
# shapes in this TP/DP group. Zero disables this rule.
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
adaptive_main "$@"

127
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh
View File

@ -1,114 +1,33 @@
#!/usr/bin/env bash
# Start vLLM server in Docker for a given TPxDP configuration.
# Start the vLLM TPxDP server through the shared deployment layer.
# Usage: start_vllm_docker.sh <TP> <DP>
#
# Uses the verified vLLM SM120 image and keeps its cache on persistent
# storage. The container is removed automatically on stop.
set -e
set -Eeuo pipefail
TP="${1}"
DP="${2}"
TP="${1:-}"
DP="${2:-}"
if [[ -z "$TP" || -z "$DP" ]]; then
echo "Usage: $0 <TP> <DP>"
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
PORT="${VLLM_PORT:-30030}"
NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
rm -f "$PID_FILE"
# Clean up any stale container with the same name.
docker rm -f "$NAME" >/dev/null 2>&1 || true
SERVER_ARGS=(
serve "$MODEL_PATH"
--trust-remote-code
--kv-cache-dtype "$KV_CACHE_DTYPE"
--block-size "$BLOCK_SIZE"
--tensor-parallel-size "$TP"
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS"
--host 0.0.0.0
--port "$PORT"
)
if [[ "$DP" -gt 1 ]]; then
SERVER_ARGS+=(
--data-parallel-size "$DP"
)
fi
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
echo "Image: $IMAGE"
echo "Model: $MODEL_PATH"
echo "Container name: $NAME"
echo "Host port: $PORT"
echo "Command: vllm ${SERVER_ARGS_STR}"
echo "Log: $LOG"
# Run docker in the foreground so that killing the host process stops the
# container (the --rm flag ensures cleanup). nohup lets us background it and
# capture the host PID in the same way as the bare-metal start script.
nohup docker run --rm \
--name "$NAME" \
--gpus all \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint vllm \
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
"$IMAGE" \
"${SERVER_ARGS[@]}" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
echo "Waiting for health on port ${PORT}..."
for i in $(seq 1 600); do
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
echo "vLLM server is ready at http://127.0.0.1:${PORT}"
echo "Log: $LOG"
exit 0
fi
# Check if the Docker container is still running (not the nohup PID).
# Allow a brief grace period for the container to appear in docker ps.
container_running=0
for _ in $(seq 1 3); do
if docker ps --filter "name=${NAME}" --format '{{.Names}}' | grep -q "^${NAME}$"; then
container_running=1
break
fi
sleep 1
done
if [[ "$container_running" -eq 0 ]]; then
echo "ERROR: Docker vLLM container exited early"
tail -200 "$LOG"
exit 1
fi
echo "Waiting... ($i/600)"
sleep 5
done
echo "ERROR: Docker vLLM server not healthy after 600 retries"
tail -200 "$LOG"
exit 1
log "starting vLLM server tp=${TP} dp=${DP} via deploy profile"
deploy_start \
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}" \
"$TP" "$DP" \
"${RUNTIME_BASE}/logs" \
"${VLLM_PORT:-30030}" \
"$MODEL_PATH" \
"${EXPERIMENT}_vllm_tp${TP}_dp${DP}"

29
experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/stop_vllm_docker.sh Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Stop the vLLM TPxDP server through the shared deployment layer.
# Usage: stop_vllm_docker.sh <TP> <DP>
set -Eeuo pipefail
TP="${1:-}"
DP="${2:-}"
if [[ -z "$TP" || -z "$DP" ]]; then
echo "Usage: $0 <TP> <DP>"
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
log "stopping vLLM server tp=${TP} dp=${DP} via deploy profile"
deploy_stop \
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}" \
"$TP" "$DP" \
"${VLLM_PORT:-30030}" \
"$MODEL_PATH" \
"${EXPERIMENT}_vllm_tp${TP}_dp${DP}"

130
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/README.md Normal file
View File

@ -0,0 +1,130 @@
# GLM-5.2-FP8 多机部署（SGLang TP=16，2×RTX 6000D 节点）
在 pro6000D.1 + pro6000D.3 两台机器（各 8×RTX 6000D）上多机部署 GLM-5.2-FP8，
并对齐 H20 / DSV4 实验的 benchmark 方法做 TP×DP 矩阵测试。
## 为什么必须多机
GLM-5.2 是 MoE 模型：**总参数约 735B，FP8 权重约 700GB**（256 路由专家 + 1 共享
专家，每 token 激活 8 个，DeepSeek-V3 架构）。单机 8 卡 RTX 6000D 显存共 685GB，
**装不下 700GB 权重**，因此 2 节点 16 卡是必需项，不是优化项。
## 架构
```
HTTP 请求 (bench_serving)
│
▼
┌──────────────┐ NCCL/RoCE (41.6GB/s) ┌──────────────┐
│ pro6000D.1 │ ◄══════════════════════► │ pro6000D.3 │
│ node-rank 0 │ 计算网 10.101/10.102 │ node-rank 1 │
│ 持有 1/2 权重 │ │ 持有 1/2 权重 │
│ HTTP :30031 │ │ 不暴露 HTTP │
└──────────────┘ └──────────────┘
```
- **node0（pro6000D.1）**：主节点，对外暴露 HTTP API，接收所有请求。
- **node1（pro6000D.3）**：从节点，只做计算，不对外服务。
- 两机通过 NCCL 连接：`--dist-init-addr <node0_ip:port> --nnodes 2 --node-rank <0|1>`。
- NCCL bootstrap 走管理网（174.1.51.x），数据面走 RoCE 计算网（mlx5_0/mlx5_3）。
- benchmark 客户端只打 node0 的 HTTP，感知不到背后是两台机器。
## 前置条件（两台机器都要满足）
1. **模型文件**：两机都要有完整的 GLM-5.2-FP8，路径统一为
`/data/hf_models/GLM-5.2-FP8`（141 个 safetensors + index.json）。
- pro6000D.1：下载中（截至编写时 112/141 分片）。
- pro6000D.3：**当前没有，需补齐**（从 .1 rsync 传过去或重新下载）。
2. **Docker 镜像**：两机都要有
`lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45`（sglang dev，原生支持
GLM-5.2 的 `glm_moe_dsa` 架构）。
- pro6000D.1：已有。
- pro6000D.3：下载中。
3. **SSH 互通**：node0 能 `ssh pro6000D.3` 免密登录（已配置）。
4. **NCCL 跨机已验证**：双 NIC 调优后 all_reduce busbw 41.6 GB/s（见带宽报告）。
## 文件说明
| 文件 | 作用 |
|------|------|
| `config.env` | 全局配置：模型路径、TP=16、上下文 256K、NCCL 调优、节点拓扑 |
| `start_sglang_node.sh <rank>` | 单节点启动（参数化 rank），node0/node1 共用 |
| `start_sglang_multinode.sh <tp> <dp>` | 多机编排：sync→起 node1→起 node0→等 health |
| `stop_sglang_multinode.sh <tp> <dp>` | 多机停止：停 node0（本地）+ node1（ssh） |
| `start_sglang_dp.sh <tp> <dp>` | run_bench 入口，委托给 multinode 编排 |
| `run_bench.sh` | TP×DP 矩阵 benchmark（复用 DSV4 模板，改 stop 为多机） |
| `matrix.json` | ISL×OSL 测试矩阵（复用 DSV4） |
| `adaptive_config.env` | 自适应并发搜索配置（复用 DSV4） |
## NCCL 调优（关键，不可省）
跨机必须带以下环境变量（已固化在 config.env，注入容器），否则带宽从 41.6
跌到 21 GB/s：
```bash
NCCL_IB_HCA=mlx5_0,mlx5_3 # 启用两张 RoCE 网卡
NCCL_MIN_NCHANNELS=8 # 强制 8 channel 分流到两卡
NCCL_IB_QPS_PER_CONNECTION=4 # 每连接 4 QP 提升 RoCE 并行
NCCL_NET_GDR_LEVEL=PHB # GPUDirect RDMA
NCCL_SOCKET_IFNAME=eth1 # bootstrap 走管理网网卡
```
## 用法
### 冒烟测试（单场景验证端到端）
```bash
cd /data/yy/sskj/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16
GRID_LIMIT=1 DRY_RUN=0 bash run_bench.sh
```
### 全矩阵（low/high 两个并发点）
```bash
CONCURRENCY_SAMPLES=2 bash run_bench.sh
```
### 手动起停（不走 bench）
```bash
bash start_sglang_dp.sh 16 1 # 起两机，等 node0 health
bash stop_sglang_multinode.sh 16 1 # 停两机
```
### 手动单独起 node1（调试）
```bash
ssh pro6000D.3 'cd /data/yy/sskj/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16 && bash start_sglang_node.sh 1'
```
## 启动流程说明
`start_sglang_multinode.sh` 做的事：
1. **rsync 实验目录到 node1**（让 node1 有 config.env + start_sglang_node.sh）
2. **ssh 后台起 node1**：`ssh pro6000D.3 '... start_sglang_node.sh 1'`，node1
启动容器后进入 rendezvous 等待窗口，等 node0 连接
3. **本地起 node0**：`start_sglang_node.sh 0`，node0 发起 NCCL 连接，两机会合后
一起加载模型
4. **轮询 node0 /health**：就绪后返回（最多等 480×5s=40 分钟，多机加载慢）
node1 的日志在远端 `/tmp/glm52_node1_inner.log` 和本实验 `runtime/logs/` 下。
## 与 H20 基线对比
- H20 基线：`experiments/h20/glm_h20_vllm_tp_dp_matrix`，单机 8 卡 vLLM TP=8。
- 本实验：2 机 16 卡 SGLang TP=16。
- **框架不同**（vLLM vs SGLang）、**并行不同**（单机 TP=8 vs 多机 TP=16），
对比时需注明。但 benchmark 方法一致（sglang bench_serving + 同一 matrix.json
+ 自适应并发），吞吐/延迟指标可对照。
- 用 `compare.py` 对比两边 results。
## 注意事项
- **单机 TP=8 装不下**：735GB 权重 ÷ 8 卡 ≈ 92GB/卡 > 85.6GB 单卡上限，会 OOM。
必须多机。
- **CUDA 版本不一致**（.1 是 12.8、.3 是 13.0）不影响：两机都在 cu130 容器内跑，
宿主机 CUDA 版本无关。
- **node1 不暴露 HTTP**：所有请求打 node0，node1 纯计算。
- **加载慢**：700GB 权重 + NCCL 建连，首次加载预计 10-20 分钟，health 轮询
上限设了 40 分钟。

56
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/adaptive_config.env Normal file
View File

@ -0,0 +1,56 @@
# Adaptive concurrency search settings.
#
# For each fixed (TP, DP, ISL, OSL), probe:
# C = start, start * multiplier, ... up to max
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
# PLATEAU_PATIENCE consecutive points.
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}"
# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
# A gain below 2% is treated as throughput saturation. Two consecutive
# low-gain points prevent one noisy measurement from stopping the search.
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
# Keep the same random workload semantics as the fixed matrix baseline.
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
# random-ids to use generated token IDs without a ShareGPT seed dataset.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
# ISL/OSL points; lower values intentionally benchmark a length distribution.
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
# Before each measured point, warm up with the same concurrency so lazy kernel
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
# cap; set a positive cap only when very high-concurrency warmup is impractical.
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
# Reject a point if the completed request count or actual token lengths do not
# match the requested workload.
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
# Optional space-separated filters, useful for smoke tests:
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
TP_LIST="${TP_LIST:-}"
ISL_LIST="${ISL_LIST:-}"
OSL_LIST="${OSL_LIST:-}"
DRY_RUN="${DRY_RUN:-0}"
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
GRID_LIMIT="${GRID_LIMIT:-0}"

162
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/compare.py Executable file
View File

@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""Cross TP×DP configuration comparison for dsv4_h200_sglang_tp_dp_matrix.
Usage:
python3 compare.py --run-root results/<run_id> [--output comparison.md]
"""
import argparse
import json
import re
from collections import defaultdict
from pathlib import Path
def load_result(result_root: Path) -> dict:
path = result_root / "results.json"
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
ttft_ok = ttft_p95_ms < ttft_limit_ms
tpot_ok = tpot_mean_ms < tpot_limit_ms
if ttft_ok and tpot_ok:
return "PASS"
if ttft_ok or tpot_ok:
return "PARTIAL"
return "FAIL"
def gpu_memory_str(gpu: dict | None) -> str:
if not gpu:
return "-"
peak = gpu.get("peak_used_mb", 0)
total = gpu.get("memory_total_mb", 0)
if total:
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
return f"{peak:.0f}"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--run-root", type=Path, required=True)
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
parser.add_argument("--ttft-limit", type=float, default=3000.0)
parser.add_argument("--tpot-limit", type=float, default=50.0)
args = parser.parse_args()
# Discover configurations: tp*_dp* directories.
configs = []
for subdir in sorted(args.run_root.iterdir()):
if not subdir.is_dir():
continue
name = subdir.name
if not (name.startswith("tp") and "_dp" in name):
continue
results_json = subdir / "results.json"
if not results_json.exists():
continue
configs.append((name, load_result(subdir)))
if not configs:
print(f"No tp*_dp* results found under {args.run_root}")
return
model = configs[0][1].get("metadata", {}).get("model", "unknown")
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
# Group by scenario name.
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
skipped: dict[str, dict[str, str]] = defaultdict(dict)
for label, data in configs:
for s in data.get("scenarios", []):
key = s["name"]
if s.get("status") == "skipped_oom":
skipped[key][label] = s.get("note", "skipped")
else:
by_scenario[key][label] = s
with open(args.output, "w", encoding="utf-8") as f:
f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n")
f.write("## Summary\n\n")
f.write(f"- Model: `{model}`\n")
f.write(f"- Hardware: {hardware}\n")
f.write("- Backend: SGLang (Docker)\n")
f.write("- Benchmark client: `sglang.benchmark.serving`\n")
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
# Configuration overview.
f.write("### Configurations\n\n")
f.write("| Config | TP | DP | GPUs/replica | Notes |\n")
f.write("|---|---:|---:|---:|---|\n")
for label, data in configs:
cfg = data.get("config", {})
tp = cfg.get("tp", "?")
dp = cfg.get("dp", "?")
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
f.write("\n")
# Side-by-side table.
f.write("## Side-by-side results\n\n")
headers = [
"Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s",
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
]
f.write("| " + " | ".join(headers) + " |\n")
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))):
_, isl, dsl = re.findall(r"\d+", scenario_name)
for label, data in configs:
s = by_scenario[scenario_name].get(label)
if s is None:
if scenario_name in skipped and label in skipped[scenario_name]:
note = skipped[scenario_name][label]
f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
continue
cfg = s["config"]
m = s["metrics"]
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
gpu = m.get("gpu_memory")
f.write(
f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | "
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
)
# Best throughput per ISL/DSL.
f.write("\n## Best throughput per (ISL, DSL)\n\n")
f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
for scenario_name, backends in by_scenario.items():
_, isl, dsl = re.findall(r"\d+", scenario_name)
isl_i, dsl_i = int(isl), int(dsl)
for label, s in backends.items():
m = s["metrics"]
out_tok = m["output_token_throughput"]
if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]:
best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s)
for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
m = s["metrics"]
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
f.write(
f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | "
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
)
f.write("\n## Notes\n\n")
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
print(f"Wrote comparison to {args.output}")
if __name__ == "__main__":
main()

145
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/config.env Executable file
View File

@ -0,0 +1,145 @@
#!/usr/bin/env bash
# Multi-node SGLang deployment for GLM-5.2-FP8 on 2x RTX 6000D nodes (16 GPUs).
#
# Topology:
# node0 = pro6000D.1 (gpu1, 174.1.51.5, 10.101.0.11 / 10.102.0.11)
# node1 = pro6000D.3 (gpu3, 174.1.51.7, 10.101.0.13 / 10.102.0.13)
# Each node has 8x RTX 6000D (85.6GB) = 685GB; GLM-5.2-FP8 is ~700GB, so a
# single node cannot hold the weights -> 2-node TP=16 is mandatory.
#
# SGLang multi-node: every node runs its own `python3 -m sglang.launch_server`
# and they connect via NCCL using --dist-init-addr/--nnodes/--node-rank.
# node0 exposes the HTTP API; node1 is compute-only. The benchmark client only
# talks to node0.
# --- Path bootstrap -------------------------------------------------------
# run_bench.sh sources platform.sh (which sets ROOT_DIR) before this file, but
# the start_* scripts source this file directly. Provide a fallback so
# ${ROOT_DIR} / ${SCRIPT_DIR} expand correctly in both paths.
SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)}"
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../../.." 2>/dev/null && pwd)}"
# If the relative climb failed (e.g. dir moved), fall back to the script dir.
[[ -z "$ROOT_DIR" || ! -d "$ROOT_DIR/platforms" ]] && ROOT_DIR="$SCRIPT_DIR"
EXPERIMENT="glm52_pro6000_sglang_multinode_tp16"
MODEL_NAME="GLM-5.2-FP8"
MODEL_PATH="/data/hf_models/GLM-5.2-FP8"
SERVED_MODEL_NAME="GLM-5.2-FP8"
SGLANG_PORT="${SGLANG_PORT:-30031}"
# Python interpreter for orchestration scripts (parse_backend.py, compare.py,
# etc.) and the benchmark client. Defaults to the system python3 if the sglang
# venv does not exist on the host.
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
# Run the benchmark client natively (0) or inside Docker (1).
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
# All 8 GPUs per node participate in TP=16 (8 per node x 2 nodes).
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
# directory under this experiment so the benchmark is self-contained.
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
# ---------------------------------------------------------------------------
# Multi-node topology
# ---------------------------------------------------------------------------
# Management network is used for the NCCL bootstrap (dist-init-addr). The data
# plane (NCCL/RoCE) is selected automatically by NCCL inside the container via
# the NCCL_* env vars below.
NODE0_HOST="${NODE0_HOST:-pro6000D.1}"
NODE1_HOST="${NODE1_HOST:-pro6000D.3}"
NODE0_IP="${NODE0_IP:-174.1.51.5}"
NODE1_IP="${NODE1_IP:-174.1.51.7}"
NNODES="${NNODES:-2}"
# Port for NCCL bootstrap (dist-init-addr). Must be free on node0.
DIST_INIT_PORT="${DIST_INIT_PORT:-50000}"
# SSH alias for reaching node1 from the control machine (node0).
NODE1_SSH="${NODE1_SSH:-root@174.1.51.7}"
# ---------------------------------------------------------------------------
# Parallel configuration. TP=16 across 2 nodes (8 GPUs/node).
# Constraint (sglang): tp_size * pp_size must be divisible by nnodes.
# 16 * 1 = 16, 16 % 2 == 0 OK
# ---------------------------------------------------------------------------
TP_SIZE="${TP_SIZE:-16}"
DP_SIZE="${DP_SIZE:-1}"
# PARALLEL_CONFIGS is kept for compatibility with the matrix runner; the single
# multi-node config is "16 1".
declare -a PARALLEL_CONFIGS=(
"16 1"
)
# ---------------------------------------------------------------------------
# SGLang server settings (aligned with dsv4_pro6000_sglang_tp_dp_matrix).
# ---------------------------------------------------------------------------
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
# TileLang DSA on SM120 currently requires BF16 KV cache. Keep the smoke-test
# context at 128K so the doubled KV-cache element size fits beside FP8 weights.
CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
# GLM-5.2 is shipped as FP8 weights; tell SGLang explicitly.
QUANTIZATION="${QUANTIZATION:-fp8}"
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-bfloat16}"
# ---------------------------------------------------------------------------
# NCCL tuning for cross-node RoCE (mandatory, otherwise bandwidth collapses).
# Verified in the bandwidth report: dual-NIC tuned all_reduce = 41.6 GB/s,
# vs 21 GB/s without these knobs.
# ---------------------------------------------------------------------------
export NCCL_IB_HCA="${NCCL_IB_HCA:-mlx5_0,mlx5_3}"
export NCCL_MIN_NCHANNELS="${NCCL_MIN_NCHANNELS:-8}"
export NCCL_IB_QPS_PER_CONNECTION="${NCCL_IB_QPS_PER_CONNECTION:-4}"
export NCCL_NET_GDR_LEVEL="${NCCL_NET_GDR_LEVEL:-PHB}"
# Bind the NCCL socket (bootstrap/OOB) to the management NIC eth1 (174.1.51.x).
# eth0/eth3 are the RoCE data-plane NICs; eth1 carries the mgmt IP.
export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth1}"
export NCCL_IB_RETRY_CNT="${NCCL_IB_RETRY_CNT:-14}"
export NCCL_IB_TIMEOUT="${NCCL_IB_TIMEOUT:-16}"
# ---------------------------------------------------------------------------
# Deployment switch. 0 = native sglang venv, 1 = Docker.
# ---------------------------------------------------------------------------
USE_DOCKER="${USE_DOCKER:-1}"
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_nightly_cu13_cache}"
# Matrix and concurrency rules are defined in matrix.json by default.
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
MATRIX_MODE="${MATRIX_MODE:-Y}"
# Sampling density for concurrency.
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
# 2 = only test the low and high endpoints.
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
# Per-scenario timeout to avoid hangs (seconds). Multi-node load is slower, so
# the default is raised.
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-2400}"
# GPU memory sampling interval (seconds).
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
# Dry-run mode: if 1, only log the server args and scenario plan without
# starting any server or sending requests.
DRY_RUN="${DRY_RUN:-0}"
# Per-config scenario limit for quick smoke tests. 0 = run all generated
# scenarios.
GRID_LIMIT="${GRID_LIMIT:-0}"
# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True,max_split_size_mb:256}"
# Seconds to wait for node1 to come up and register before starting node0.
NODE1_READINESS_WAIT_S="${NODE1_READINESS_WAIT_S:-15}"

3321
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_backend.py Normal file
View File

File diff suppressed because it is too large Load Diff

2540
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_indexer.py Normal file
View File

File diff suppressed because it is too large Load Diff

98
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/generate_scenarios.py Executable file
View File

@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Generate the scenario list for the TP×DP matrix experiment.
Reads matrix.json and prints TSV lines:
mark input_len output_len concurrency num_prompts
mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each.
"""
import argparse
import json
import math
import os
from pathlib import Path
def sample_concurrency(low: int, high: int, target: int) -> list[int]:
"""Return only the low and high concurrency values in [low, high].
For the TP×DP matrix we only need the two endpoints of the concurrency
range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for
API compatibility but is ignored.
"""
assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
if low == high:
return [low]
return [low, high]
def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
with open(matrix_path, "r", encoding="utf-8") as f:
data = json.load(f)
matrix = data["matrix"]
concurrency_cfg = data["concurrency"]
scenarios = []
for isl_str in sorted(matrix.keys(), key=int):
dsl_map = matrix[isl_str]
low = concurrency_cfg[isl_str]["low"]
high = concurrency_cfg[isl_str]["high"]
concurrencies = sample_concurrency(low, high, target_samples)
for dsl_str in sorted(dsl_map.keys(), key=int):
mark = dsl_map[dsl_str]
if mode == "Y" and mark != "Y":
continue
if mode == "Y+P" and mark not in ("Y", "P"):
continue
# mode == "all" keeps everything, including N.
for conc in concurrencies:
scenarios.append(
{
"mark": mark,
"input_len": int(isl_str),
"output_len": int(dsl_str),
"concurrency": conc,
"num_prompts": conc * 5,
}
)
return scenarios
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--matrix", type=Path, default=Path("matrix.json"))
parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None,
help="Scenario selection mode. Defaults to matrix.mode.")
parser.add_argument("--target-samples", type=int, default=0,
help="Target number of concurrency samples. 0 = heuristic (6-8).")
args = parser.parse_args()
with open(args.matrix, "r", encoding="utf-8") as f:
data = json.load(f)
mode = args.mode if args.mode else data.get("mode", "Y+P")
target_samples = args.target_samples
if target_samples <= 0:
env_samples = os.getenv("CONCURRENCY_SAMPLES", "0")
try:
target_samples = int(env_samples)
except ValueError:
target_samples = 0
if target_samples <= 0:
target_samples = 7
scenarios = generate_scenarios(args.matrix, mode, target_samples)
print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts")
for s in scenarios:
print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}")
if __name__ == "__main__":
main()

128
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/matrix.json Normal file
View File

@ -0,0 +1,128 @@
{
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
"mode": "Y",
"matrix": {
"1024": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"4096": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"8192": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "Y"
},
"16384": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"32768": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "Y",
"4096": "P"
},
"65536": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "Y",
"2048": "P",
"4096": "N"
},
"131072": {
"128": "Y",
"256": "Y",
"512": "Y",
"1024": "P",
"2048": "N",
"4096": "N"
},
"262144": {
"128": "Y",
"256": "Y",
"512": "P",
"1024": "N",
"2048": "N",
"4096": "N"
},
"524288": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
},
"1048576": {
"128": "Y",
"256": "P",
"512": "N",
"1024": "N",
"2048": "N",
"4096": "N"
}
},
"concurrency": {
"1024": {
"low": 1,
"high": 128
},
"4096": {
"low": 1,
"high": 64
},
"8192": {
"low": 1,
"high": 64
},
"16384": {
"low": 1,
"high": 32
},
"32768": {
"low": 1,
"high": 16
},
"65536": {
"low": 1,
"high": 8
},
"131072": {
"low": 1,
"high": 4
},
"262144": {
"low": 1,
"high": 2
},
"524288": {
"low": 1,
"high": 2
},
"1048576": {
"low": 1,
"high": 2
}
}
}

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-134240/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-134240/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-134240",
"timestamp": "2026-07-24T13:42:41+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-134240/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-143107/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-143107/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-143107",
"timestamp": "2026-07-24T14:31:22+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-143107/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-150539/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-150539",
"timestamp": "2026-07-24T15:05:40+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-150539/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-151233/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-151233/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-151233",
"timestamp": "2026-07-24T15:12:34+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-151233/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-152532/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-152532/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-152532",
"timestamp": "2026-07-24T15:25:33+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-152532/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-153034/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-153034/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-153034",
"timestamp": "2026-07-24T15:30:34+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-153034/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-154223/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-154223/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-154223",
"timestamp": "2026-07-24T15:42:24+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-154223/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-154918/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,20 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-154918",
"timestamp": "2026-07-24T15:49:19+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": []
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-154918/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-160131/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,20 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-160131",
"timestamp": "2026-07-24T16:01:33+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": []
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-160131/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161611/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161611/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-161611",
"timestamp": "2026-07-24T16:16:12+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161611/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161700/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161700/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-161700",
"timestamp": "2026-07-24T16:17:00+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-161700/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

20
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-164734/comparison.md Normal file
View File

@ -0,0 +1,20 @@
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
## Summary
- Model: `/data/hf_models/GLM-5.2-FP8`
- Hardware: 8x NVIDIA RTX 6000D 84GB
- Backend: SGLang (Docker)
- Benchmark client: `sglang.benchmark.serving`
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
### Configurations
| Config | TP | DP | GPUs/replica | Notes |
|---|---:|---:|---:|---|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
## Side-by-side results
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|

489
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-164734/tp16_dp1/results.json Normal file
View File

@ -0,0 +1,489 @@
{
"metadata": {
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
"run_id": "20260724-164734",
"timestamp": "2026-07-24T16:47:35+08:00",
"model": "/data/hf_models/GLM-5.2-FP8",
"backend": "sglang",
"engine": "sglang",
"hardware": "8x NVIDIA RTX 6000D 84GB",
"accelerator": "NVIDIA RTX 6000D",
"chip": "nvidia_rtx6000d",
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
"git_commit": "d13f61f",
"git_dirty": "dirty",
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
},
"config": {},
"scenarios": [
{
"name": "c1_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c128_i1024_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i4096_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c64_i8192_o4096",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c32_i16384_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c16_i32768_o2048",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c8_i65536_o1024",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c4_i131072_o512",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i262144_o256",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i524288_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c1_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
},
{
"name": "c2_i1048576_o128",
"config": "",
"status": "SKIPPED_SERVICE_START_FAILED",
"note": "service failed to start"
}
]
}

79
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/results/20260724-164734/tp16_dp1/scenarios.tsv Normal file
View File

@ -0,0 +1,79 @@
mark input_len output_len concurrency num_prompts
Y 1024 128 1 5
Y 1024 128 128 640
Y 1024 256 1 5
Y 1024 256 128 640
Y 1024 512 1 5
Y 1024 512 128 640
Y 1024 1024 1 5
Y 1024 1024 128 640
Y 1024 2048 1 5
Y 1024 2048 128 640
Y 1024 4096 1 5
Y 1024 4096 128 640
Y 4096 128 1 5
Y 4096 128 64 320
Y 4096 256 1 5
Y 4096 256 64 320
Y 4096 512 1 5
Y 4096 512 64 320
Y 4096 1024 1 5
Y 4096 1024 64 320
Y 4096 2048 1 5
Y 4096 2048 64 320
Y 4096 4096 1 5
Y 4096 4096 64 320
Y 8192 128 1 5
Y 8192 128 64 320
Y 8192 256 1 5
Y 8192 256 64 320
Y 8192 512 1 5
Y 8192 512 64 320
Y 8192 1024 1 5
Y 8192 1024 64 320
Y 8192 2048 1 5
Y 8192 2048 64 320
Y 8192 4096 1 5
Y 8192 4096 64 320
Y 16384 128 1 5
Y 16384 128 32 160
Y 16384 256 1 5
Y 16384 256 32 160
Y 16384 512 1 5
Y 16384 512 32 160
Y 16384 1024 1 5
Y 16384 1024 32 160
Y 16384 2048 1 5
Y 16384 2048 32 160
Y 32768 128 1 5
Y 32768 128 16 80
Y 32768 256 1 5
Y 32768 256 16 80
Y 32768 512 1 5
Y 32768 512 16 80
Y 32768 1024 1 5
Y 32768 1024 16 80
Y 32768 2048 1 5
Y 32768 2048 16 80
Y 65536 128 1 5
Y 65536 128 8 40
Y 65536 256 1 5
Y 65536 256 8 40
Y 65536 512 1 5
Y 65536 512 8 40
Y 65536 1024 1 5
Y 65536 1024 8 40
Y 131072 128 1 5
Y 131072 128 4 20
Y 131072 256 1 5
Y 131072 256 4 20
Y 131072 512 1 5
Y 131072 512 4 20
Y 262144 128 1 5
Y 262144 128 2 10
Y 262144 256 1 5
Y 262144 256 2 10
Y 524288 128 1 5
Y 524288 128 2 10
Y 1048576 128 1 5
Y 1048576 128 2 10
1 mark input_len output_len concurrency num_prompts
2 Y 1024 128 1 5
3 Y 1024 128 128 640
4 Y 1024 256 1 5
5 Y 1024 256 128 640
6 Y 1024 512 1 5
7 Y 1024 512 128 640
8 Y 1024 1024 1 5
9 Y 1024 1024 128 640
10 Y 1024 2048 1 5
11 Y 1024 2048 128 640
12 Y 1024 4096 1 5
13 Y 1024 4096 128 640
14 Y 4096 128 1 5
15 Y 4096 128 64 320
16 Y 4096 256 1 5
17 Y 4096 256 64 320
18 Y 4096 512 1 5
19 Y 4096 512 64 320
20 Y 4096 1024 1 5
21 Y 4096 1024 64 320
22 Y 4096 2048 1 5
23 Y 4096 2048 64 320
24 Y 4096 4096 1 5
25 Y 4096 4096 64 320
26 Y 8192 128 1 5
27 Y 8192 128 64 320
28 Y 8192 256 1 5
29 Y 8192 256 64 320
30 Y 8192 512 1 5
31 Y 8192 512 64 320
32 Y 8192 1024 1 5
33 Y 8192 1024 64 320
34 Y 8192 2048 1 5
35 Y 8192 2048 64 320
36 Y 8192 4096 1 5
37 Y 8192 4096 64 320
38 Y 16384 128 1 5
39 Y 16384 128 32 160
40 Y 16384 256 1 5
41 Y 16384 256 32 160
42 Y 16384 512 1 5
43 Y 16384 512 32 160
44 Y 16384 1024 1 5
45 Y 16384 1024 32 160
46 Y 16384 2048 1 5
47 Y 16384 2048 32 160
48 Y 32768 128 1 5
49 Y 32768 128 16 80
50 Y 32768 256 1 5
51 Y 32768 256 16 80
52 Y 32768 512 1 5
53 Y 32768 512 16 80
54 Y 32768 1024 1 5
55 Y 32768 1024 16 80
56 Y 32768 2048 1 5
57 Y 32768 2048 16 80
58 Y 65536 128 1 5
59 Y 65536 128 8 40
60 Y 65536 256 1 5
61 Y 65536 256 8 40
62 Y 65536 512 1 5
63 Y 65536 512 8 40
64 Y 65536 1024 1 5
65 Y 65536 1024 8 40
66 Y 131072 128 1 5
67 Y 131072 128 4 20
68 Y 131072 256 1 5
69 Y 131072 256 4 20
70 Y 131072 512 1 5
71 Y 131072 512 4 20
72 Y 262144 128 1 5
73 Y 262144 128 2 10
74 Y 262144 256 1 5
75 Y 262144 256 2 10
76 Y 524288 128 1 5
77 Y 524288 128 2 10
78 Y 1048576 128 1 5
79 Y 1048576 128 2 10

154
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/run_adaptive_concurrency.sh Executable file
View File

@ -0,0 +1,154 @@
#!/usr/bin/env bash
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT"
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
ACTIVE_ENGINE_SERVER_LOG=""
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
PYTHON="${VENV_CLIENT}/bin/python"
else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
}
engine_stop_server() {
local tp="$1"
local dp="$2"
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
engine_build_server_args() {
local tp="$1"
local dp="$2"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {
local tp="$1"
local dp="$2"
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
log "starting sglang server tp=${tp} dp=${dp}"
if [[ -n "${CONTAINER_NAME:-}" ]]; then
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
else
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
fi
if ! engine_is_healthy; then
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
return 1
fi
if [[ -z "${CONTAINER_NAME:-}" ]]; then
ACTIVE_ENGINE_SERVER_LOG="$(
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
)"
fi
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
}
engine_detect_oom() {
local detail_log="$1"
local tp="$2"
local dp="$3"
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
local -a logs=("$detail_log" "$outer_log")
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
fi
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
return 0
fi
if [[ -n "${CONTAINER_NAME:-}" ]]; then
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
return $?
fi
return 1
}
engine_run_bench() {
local isl="$1"
local osl="$2"
local concurrency="$3"
local num_prompts="$4"
local output_file="$5"
local warmup_requests
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
local -a bench_args=(
--backend sglang
--host 127.0.0.1
--port "$ENGINE_PORT"
--dataset-name "$BENCH_DATASET_NAME"
--random-input-len "$isl"
--random-output-len "$osl"
--random-range-ratio "$RANDOM_RANGE_RATIO"
--num-prompts "$num_prompts"
--max-concurrency "$concurrency"
--request-rate 10000
--warmup-requests "$warmup_requests"
--output-file "$output_file"
--output-details
--disable-tqdm
)
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
bench_args+=(--dataset-path "$DATASET_PATH")
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
: # random-ids does not need --tokenize-prompt
else
bench_args+=(--tokenize-prompt)
fi
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
fi
docker run --rm \
--network host \
"${volume_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"$DOCKER_IMAGE" \
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"

163
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/run_adaptive_concurrency_add16.sh Executable file
View File

@ -0,0 +1,163 @@
#!/usr/bin/env bash
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT"
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
ACTIVE_ENGINE_SERVER_LOG=""
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
PYTHON="${VENV_CLIENT}/bin/python"
else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
engine_is_healthy() {
curl --fail --silent --show-error --max-time 5 \
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
}
engine_stop_server() {
local tp="$1"
local dp="$2"
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
ACTIVE_ENGINE_SERVER_LOG=""
sleep 2
}
engine_build_server_args() {
local tp="$1"
local dp="$2"
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
}
engine_start_server() {
local tp="$1"
local dp="$2"
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
log "starting sglang server tp=${tp} dp=${dp}"
if [[ -n "${CONTAINER_NAME:-}" ]]; then
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
else
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
fi
if ! engine_is_healthy; then
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
return 1
fi
if [[ -z "${CONTAINER_NAME:-}" ]]; then
ACTIVE_ENGINE_SERVER_LOG="$(
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
)"
fi
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
}
engine_detect_oom() {
local detail_log="$1"
local tp="$2"
local dp="$3"
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
local -a logs=("$detail_log" "$outer_log")
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
fi
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
return 0
fi
if [[ -n "${CONTAINER_NAME:-}" ]]; then
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
return $?
fi
return 1
}
engine_run_bench() {
local isl="$1"
local osl="$2"
local concurrency="$3"
local num_prompts="$4"
local output_file="$5"
local warmup_requests
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
local -a bench_args=(
--backend sglang
--host 127.0.0.1
--port "$ENGINE_PORT"
--dataset-name "$BENCH_DATASET_NAME"
--random-input-len "$isl"
--random-output-len "$osl"
--random-range-ratio "$RANDOM_RANGE_RATIO"
--num-prompts "$num_prompts"
--max-concurrency "$concurrency"
--request-rate 10000
--warmup-requests "$warmup_requests"
--output-file "$output_file"
--output-details
--disable-tqdm
)
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
bench_args+=(--dataset-path "$DATASET_PATH")
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
: # random-ids does not need --tokenize-prompt
else
bench_args+=(--tokenize-prompt)
fi
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
fi
docker run --rm \
--network host \
"${volume_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"$DOCKER_IMAGE" \
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
else
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
fi
}
export -f engine_run_bench
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
adaptive_main "$@"
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
# shapes in this TP/DP group. Zero disables this rule.
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"

536
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/run_bench.sh Executable file
View File

@ -0,0 +1,536 @@
#!/usr/bin/env bash
# TP×DP matrix benchmark for DeepSeek-V4-Flash on SGLang (Docker).
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_BASE="${SCRIPT_DIR}/results"
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
MATRIX_MODE="${MATRIX_MODE:-Y}"
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
DRY_RUN="${DRY_RUN:-0}"
GRID_LIMIT="${GRID_LIMIT:-0}"
# Export variables used inside functions that are called via bash -c subshells.
export BENCH_DATASET_NAME DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT SGLANG_BENCH_MODULE
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
PYTHON="${VENV_CLIENT}/bin/python"
else
PYTHON="$(command -v python3)"
fi
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
mkdir -p "$log_dir_global"
log_init "${log_dir_global}/orchestrator.log"
log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
is_server_healthy() {
curl --fail --silent --show-error --max-time 10 "http://127.0.0.1:${SGLANG_PORT}/v1/models" >/dev/null 2>&1
}
stop_server() {
local tp="$1"
local dp="$2"
log "stopping multi-node sglang (tp=${tp}, dp=${dp}) on both nodes"
# Delegate to the multi-node stop orchestrator, which tears down node0
# (local) and node1 (remote via SSH) together: pid file, docker rm, pkill.
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" \
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.stop.log" 2>&1 || true
}
build_server_args() {
local tp="$1"
local dp="$2"
local args=(
"python3 -m sglang.launch_server" --model-path "$MODEL_PATH"
--trust-remote-code
--tp-size "$tp"
--dp-size "$dp"
--quantization "$QUANTIZATION"
--moe-runner-backend "$MOE_RUNNER_BACKEND"
--mem-fraction-static "$MEM_FRACTION_STATIC"
--context-length "$CONTEXT_LENGTH"
--max-running-requests "$MAX_RUNNING_REQUESTS"
--host 0.0.0.0
--port "$SGLANG_PORT"
--dist-init-addr "${NODE0_IP}:${DIST_INIT_PORT}"
--nnodes "$NNODES"
--node-rank 0
)
printf '%s ' "${args[@]}"
}
start_server() {
local tp="$1"
local dp="$2"
log "starting sglang server tp=${tp} dp=${dp}"
if [[ -n "${CONTAINER_NAME:-}" ]]; then
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" \
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1
else
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1
fi
if ! is_server_healthy; then
log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}"
return 1
fi
log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}"
}
restart_server() {
local tp="$1"
local dp="$2"
log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure"
stop_server "$tp" "$dp"
sleep 10
start_server "$tp" "$dp"
}
run_bench_serving() {
# Inject the offline workload choice consistently for warmup and main runs.
local -a dataset_args=(--dataset-name "$BENCH_DATASET_NAME")
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
dataset_args+=(--dataset-path "$DATASET_PATH")
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
: # random-ids does not need --tokenize-prompt
else
dataset_args+=(--tokenize-prompt)
fi
if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then
local vol_args=()
vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro")
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro")
fi
vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}")
docker run --rm \
--network host \
"${vol_args[@]}" \
-e PYTHONUNBUFFERED=1 \
--entrypoint python3 \
"${DOCKER_IMAGE}" \
-m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
else
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
fi
}
export -f run_bench_serving
run_warmup() {
local input_len="$1"
local output_len="$2"
log "warming up (input=${input_len}, output=${output_len}, num=1)"
bash -c '
run_bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port "'"$SGLANG_PORT"'" \
--random-input-len "'"$input_len"'" \
--random-output-len "'"$output_len"'" \
--num-prompts 1 \
--max-concurrency 1 \
--request-rate 10000 \
--output-file /dev/null \
--output-details \
>> "'"${log_dir_global}/warmup.log"'" 2>&1
'
log "warmup completed"
}
scenario_already_completed() {
local output_file="$1"
local expected="$2"
[[ -s "$output_file" ]] || return 1
local completed
completed="$("$PYTHON" -c "
import json, sys
path = sys.argv[1]
try:
with open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
data = json.loads(line)
print(data.get('completed', 0))
break
except Exception:
print(0)
" "$output_file")"
[[ "${completed:-0}" -ge "$expected" ]]
}
scenario_already_processed() {
local result_root="$1"
local scenario_name="$2"
local json_path="${result_root}/results.json"
[[ -f "$json_path" ]] || return 1
"$PYTHON" -c "
import json, sys
path, name = sys.argv[1], sys.argv[2]
try:
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
for s in data.get('scenarios', []):
if s.get('name') == name:
if s.get('status') or s.get('metrics', {}).get('success', 0) > 0:
sys.exit(0)
except Exception:
pass
sys.exit(1)
" "$json_path" "$scenario_name"
}
detect_oom() {
local detail_log="$1"
local server_outer_log="$2"
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then
return 0
fi
return 1
}
start_gpu_monitor() {
local csv_path="$1"
mkdir -p "$(dirname "$csv_path")"
nvidia-smi \
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \
--format=csv \
-l "$GPU_MEM_SAMPLE_INTERVAL_S" \
> "$csv_path" 2>/dev/null &
echo $!
}
stop_gpu_monitor() {
local pid="$1"
if kill -0 "$pid" 2>/dev/null; then
kill "$pid" 2>/dev/null || true
sleep 1
kill -9 "$pid" 2>/dev/null || true
fi
}
append_scenario_record() {
local result_root="$1"
local json_path="$result_root/results.json"
shift
local scenario_json
scenario_json="$("$PYTHON" -c "
import json, sys
pairs = [a.split('=', 1) for a in sys.argv[1:]]
d = {}
for k, v in pairs:
try:
d[k] = json.loads(v)
except json.JSONDecodeError:
d[k] = v
print(json.dumps(d, ensure_ascii=False))
" "$@")"
PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
}
record_skipped_csv() {
local csv_path="$1"
shift
# Args: key=value
local row
row="$("$PYTHON" -c "
import csv, json, sys, io
pairs = [a.split('=', 1) for a in sys.argv[1:]]
d = {}
for k, v in pairs:
try:
d[k] = json.loads(v)
except json.JSONDecodeError:
d[k] = v
buf = io.StringIO()
writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore')
writer.writerow(d)
print(buf.getvalue().strip())
" "$@")"
echo "$row" >> "$csv_path"
}
skip_remaining_scenarios() {
local result_root="$1"
local scenario_tsv="$2"
local start_index="$3"
local status="$4"
local reason="$5"
local tp="$6"
local dp="$7"
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
local i=0
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
if (( i < start_index )); then
i=$((i + 1))
continue
fi
i=$((i + 1))
local sname="c${conc}_i${isl}_o${dsl}"
if scenario_already_processed "$result_root" "$sname"; then
continue
fi
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"${status}\"" \
"note=\"${reason}\""
record_skipped_csv "$skipped_csv" \
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}"
done
}
# ---------------------------------------------------------------------------
# Per-configuration runner
# ---------------------------------------------------------------------------
run_parallel_config() {
local tp="$1"
local dp="$2"
local config_label="tp${tp}_dp${dp}"
local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}"
local raw_dir="${result_root}/raw_outputs"
local gpu_log_dir="${result_root}/gpu_logs"
local phase_log_dir="${result_root}/logs"
mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir"
log "===== ${config_label} START ====="
# Generate scenario list for this config.
local scenario_tsv="${result_root}/scenarios.tsv"
"$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \
--matrix "$MATRIX_FILE" \
--mode "$MATRIX_MODE" \
> "$scenario_tsv"
local total_scenarios
total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
log "generated ${total_scenarios} scenarios for ${config_label}"
# Write metadata.
ensure_result_root "$result_root"
write_metadata_json \
"${result_root}/results.json" \
"${EXPERIMENT_NAME}_${config_label}" \
"$RUN_ID" \
"$MODEL_PATH" \
"sglang" \
"sglang" \
"$HARDWARE" \
"$ACCELERATOR" \
"$CHIP" \
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
"$DOCKER_IMAGE" \
"H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
local server_args_str
server_args_str="$(build_server_args "$tp" "$dp")"
jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \
'.config = {
"tp": ($tp | tonumber),
"dp": ($dp | tonumber),
"cuda_visible_devices": $cuda,
"backend": "sglang",
"server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_sglang_dp.sh",
"server_args": $args
}' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \
mv "${result_root}/results.json.tmp" "${result_root}/results.json"
if [[ "$DRY_RUN" == "1" ]]; then
log "DRY_RUN: would start server with args: ${server_args_str}"
local line
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
done
log "===== ${config_label} DONE (dry run) ====="
return 0
fi
# Initialize skipped_after_oom.csv for this run.
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
if [[ ! -f "$skipped_csv" ]]; then
echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv"
fi
# Start server once for this TP×DP config.
if ! start_server "$tp" "$dp"; then
log "ERROR: ${config_label} failed to start; skipping all scenarios"
skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp"
log "===== ${config_label} DONE ====="
return 0
fi
# Warmup with a small prompt before the first scenario.
run_warmup 1024 128 || true
# Read scenarios into an array so we can skip remaining entries on failure.
local -a scenarios=()
while IFS= read -r line; do
scenarios+=("$line")
done < <(tail -n +2 "$scenario_tsv")
local i mark isl dsl conc num
local output_file detail_log gpu_csv sname bench_rc
for (( i = 0; i < ${#scenarios[@]}; i++ )); do
IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}"
if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then
log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios"
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp"
break
fi
sname="c${conc}_i${isl}_o${dsl}"
output_file="${raw_dir}/sglang_main_${conc}_${isl}_${dsl}.jsonl"
detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log"
gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv"
if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then
log "skipping already-processed ${config_label} scenario: ${sname}"
continue
fi
log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
local gpu_pid
gpu_pid="$(start_gpu_monitor "$gpu_csv")"
bench_rc=0
timeout "$SCENARIO_TIMEOUT_S" bash -c '
run_bench_serving \
--backend sglang \
--host 127.0.0.1 \
--port "'"$SGLANG_PORT"'" \
--random-input-len "'"$isl"'" \
--random-output-len "'"$dsl"'" \
--num-prompts "'"$num"'" \
--max-concurrency "'"$conc"'" \
--request-rate 10000 \
--output-file "'"$output_file"'" \
--output-details \
> "'"$detail_log"'" 2>&1
' || bench_rc=$?
stop_gpu_monitor "$gpu_pid"
if [[ "$bench_rc" -eq 0 ]]; then
log "finished ${config_label} scenario: output=${output_file}"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"completed\"" \
"note=\"benchmark finished successfully\""
continue
fi
# Failure handling.
if detect_oom "$detail_log" "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log"; then
log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"OOM\"" \
"note=\"detected CUDA out-of-memory\""
record_skipped_csv "$skipped_csv" \
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}"
stop_server "$tp" "$dp"
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp"
break
fi
log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}"
if [[ "$mark" == "P" ]]; then
log "optional (P) scenario failed; recording as skipped and continuing"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"skipped_optional\"" \
"note=\"optional scenario failed (rc=${bench_rc})\""
record_skipped_csv "$skipped_csv" \
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}"
continue
fi
# Mandatory scenario failed but not OOM: try to restart the server.
if restart_server "$tp" "$dp"; then
run_warmup 1024 128 || true
log "resuming ${config_label} after server restart"
continue
fi
log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
append_scenario_record "$result_root" \
"name=${sname}" \
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
"status=\"FAILED\"" \
"note=\"scenario failed and server restart failed (rc=${bench_rc})\""
record_skipped_csv "$skipped_csv" \
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}"
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp"
break
done
stop_server "$tp" "$dp"
# Parse results.
log "parsing ${config_label} results"
"$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \
>> "${phase_log_dir}/parse.log" 2>&1 || {
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
}
log "===== ${config_label} DONE ====="
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
# Cleanup any leftovers.
for cfg in "${PARALLEL_CONFIGS[@]}"; do
read -r tp dp <<< "$cfg"
stop_server "$tp" "$dp"
done
# Run each parallel configuration.
for cfg in "${PARALLEL_CONFIGS[@]}"; do
read -r tp dp <<< "$cfg"
run_parallel_config "$tp" "$dp"
done
# Generate cross-configuration comparison.
log "generating comparison report"
"$PYTHON" "${SCRIPT_DIR}/compare.py" \
--run-root "${RESULT_BASE}/${RUN_ID}" \
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
>> "${log_dir_global}/compare.log" 2>&1 || {
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
}
log "all results saved to ${RESULT_BASE}/${RUN_ID}"

16
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_dp.sh Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Entry point for the matrix runner (run_bench.sh calls this as
# start_sglang_dp.sh <tp> <dp>). For this multi-node experiment it always
# delegates to the multi-node orchestrator, which brings up both nodes.
#
# Usage: start_sglang_dp.sh <tp> <dp>
set -e
TP="${1:?usage: start_sglang_dp.sh <tp> <dp>}"
DP="${2:?usage: start_sglang_dp.sh <tp> <dp>}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# This experiment is multi-node only; there is no single-node venv path.
# USE_DOCKER is kept for config.env compatibility but ignored here.
exec "${SCRIPT_DIR}/start_sglang_multinode.sh" "$TP" "$DP"

45
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_multinode.sh Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Multi-node orchestrator for GLM-5.2-FP8 TP=16 across pro6000D.1 + pro6000D.3,
# through the shared deployment layer (`python -m sskj.deploy start` with
# NODE_HOSTS orchestration). Rank 0 (local) is started without ssh.
#
# Usage: start_sglang_multinode.sh <tp> <dp>
set -e
TP="${1:-${TP_SIZE:-16}}"
DP="${2:-${DP_SIZE:-1}}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
DRY_RUN="${DRY_RUN:-0}"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
echo "=== Multi-node start: TP=${TP} DP=${DP} nnodes=${NNODES} via deploy profile ${DEPLOY_PROFILE} ==="
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP" --dp "$DP" \
--port "${SGLANG_PORT:-30031}" \
--model-path "$MODEL_PATH" \
--log-dir "${RUNTIME_BASE}/logs" \
--dry-run
else
deploy_start_multinode \
"$DEPLOY_PROFILE" "$TP" "$DP" \
"${RUNTIME_BASE}/logs" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi

46
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_node.sh Executable file
View File

@ -0,0 +1,46 @@
#!/usr/bin/env bash
# Start one GLM-5.2-FP8 SGLang node through the shared deployment layer.
# Convenience for per-node debugging; the multinode orchestrator
# (start_sglang_multinode.sh) does not use this anymore.
#
# Usage: start_sglang_node.sh <node_rank>
set -e
NODE_RANK="${1:?usage: start_sglang_node.sh <node_rank>}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
DRY_RUN="${DRY_RUN:-0}"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
echo "=== Start node${NODE_RANK} (TP=${TP_SIZE}, DP=${DP_SIZE}, nnodes=${NNODES}) via deploy profile ==="
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "${TP_SIZE:-16}" --dp "${DP_SIZE:-1}" \
--port "${SGLANG_PORT:-30031}" \
--model-path "$MODEL_PATH" \
--node-rank "$NODE_RANK" \
--log-dir "${RUNTIME_BASE}/logs" \
--dry-run
else
deploy_start \
"$DEPLOY_PROFILE" "${TP_SIZE:-16}" "${DP_SIZE:-1}" \
"${RUNTIME_BASE}/logs" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH" \
"$EXPERIMENT" \
"$NODE_RANK"
fi

45
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/stop_sglang_multinode.sh Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Tear down the multi-node GLM-5.2-FP8 deployment (both nodes) through the
# shared deployment layer.
#
# Usage: stop_sglang_multinode.sh <tp> <dp>
set -e
TP="${1:-${TP_SIZE:-16}}"
DP="${2:-${DP_SIZE:-1}}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
DRY_RUN="${DRY_RUN:-0}"
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
echo "=== Multi-node stop: TP=${TP} DP=${DP} via deploy profile ${DEPLOY_PROFILE} ==="
if [[ "$DRY_RUN" == "1" ]]; then
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
--tp "$TP" --dp "$DP" \
--port "${SGLANG_PORT:-30031}" \
--model-path "$MODEL_PATH" \
--container-name "$EXPERIMENT" \
--dry-run
else
deploy_stop \
"$DEPLOY_PROFILE" "$TP" "$DP" \
"${SGLANG_PORT:-30031}" \
"$MODEL_PATH" \
"$EXPERIMENT"
fi
# Local fallback: kill any stray launch_server for this model.
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
pkill -9 -f "sglang.launch_server.*${MODEL_PATH}" 2>/dev/null || true
echo "=== both nodes stopped ==="

72
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/test_glm52_tilelang_partial.py Normal file
View File

@ -0,0 +1,72 @@
import math
import os
import torch
from sglang.kernels.ops.attention.dsa.tilelang_kernel import (
sparse_mla_fwd_decode_combine,
sparse_mla_fwd_decode_partial,
tilelang_sparse_fwd,
)
def main() -> None:
torch.manual_seed(0)
heads = int(os.environ.get("HEADS", "4"))
d_v = 256
d_tail = 64
topk = 2048
block_i = 32
threads = 128
scale = 1.0 / math.sqrt(d_v + d_tail)
q = torch.randn((1, heads, d_v + d_tail), device="cuda", dtype=torch.bfloat16)
kv = torch.randn(
(topk, 1, d_v + d_tail), device="cuda", dtype=torch.bfloat16
)
indices = torch.arange(topk, device="cuda", dtype=torch.int32).view(1, 1, topk)
if os.environ.get("USE_WRAPPER") == "1":
output = tilelang_sparse_fwd(q, kv, indices, scale, d_v)
else:
partial_kernel = sparse_mla_fwd_decode_partial(
heads,
d_v,
d_tail,
topk,
sm_scale=scale,
block_I=block_i,
inner_iter=1,
num_stages=1,
threads=threads,
)
partial_o, partial_lse = partial_kernel(
q.unsqueeze(0), kv.unsqueeze(0), indices.unsqueeze(0)
)
groups = topk // block_i
combine_kernel = sparse_mla_fwd_decode_combine(
heads,
d_v,
groups * block_i,
head_per_block=4,
block_I=block_i,
threads=threads,
)
output = combine_kernel(partial_o, partial_lse)
q_float = q[0].float()
k_float = kv[:, 0].float()
scores = q_float @ k_float.transpose(0, 1) * scale
probs = torch.softmax(scores, dim=-1)
reference = probs @ k_float[:, :d_v]
actual = output.reshape(heads, d_v).float()
error = (actual - reference).abs()
print(f"output_shape={tuple(output.shape)}")
print(f"max_abs_error={error.max().item():.6f}")
print(f"mean_abs_error={error.mean().item():.6f}")
print(f"all_finite={torch.isfinite(actual).all().item()}")
if __name__ == "__main__":
main()

2610
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/tilelang_kernel_sm120.py Normal file
View File

File diff suppressed because it is too large Load Diff

2
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/config.env
View File

@ -17,6 +17,8 @@ DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_clean
# Serving port + container.
PORT="${PORT:-30010}"
# Alias for the unified bench layer (`python -m sskj.bench`).
SGLANG_PORT="${SGLANG_PORT:-${PORT:-30010}}"
CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_pro6000_sglang_tp8}"
# Docker image: sm120-capable SGLang (built/verified on this RTX 6000D box),

10
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/run_bench.sh
View File

@ -85,14 +85,8 @@ fi
stop_server() {
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
log "stopping server (kill docker run PID -> --rm removes container)"
if [[ -f "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid" ]]; then
local pid
pid="$(cat "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid")"
kill "$pid" 2>/dev/null || true
sleep 3
kill -9 "$pid" 2>/dev/null || true
fi
log "stopping server via deploy profile"
bash "${SCRIPT_DIR}/stop_server.sh" >> "${LOG_DIR}/stop_server.outer.log" 2>&1 || true
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
fi
}

85
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/start_server.sh
View File

@ -1,92 +1,31 @@
#!/usr/bin/env bash
# Start the SGLang server for Qwen3-235B-A22B on NVIDIA RTX 6000D (TP=8, 8 GPUs).
#
# NVIDIA counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh.
# Uses the standard sglang qwen3_moe path (no Kunlun patch, no XPU env).
# Launch pattern mirrors pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh.
# Start the Qwen3-235B-A22B SGLang TP=8 server through the shared deployment layer.
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# Source config before platform so our CONTAINER_NAME/PORT defaults win.
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
LOG_DIR="${RESULT_ROOT}/logs"
mkdir -p "$LOG_DIR" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
log_init "${LOG_DIR}/start_server.log"
log "starting server for ${EXPERIMENT_NAME} (NVIDIA, TP=${TP})"
log "starting server for ${EXPERIMENT_NAME} (NVIDIA, TP=${TP}) via deploy profile"
log "model: ${MODEL_PATH}"
log "image: ${DOCKER_IMAGE}"
log "port: ${PORT} container: ${CONTAINER_NAME}"
SERVER_LOG="${LOG_DIR}/server.log"
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid"
rm -f "$PID_FILE"
# Remove any stale container with the same name.
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
# Standard sglang launch for qwen3_moe on NVIDIA (bf16 auto, flashinfer attn).
# Mount /data so model + experiment results dir + dataset are all at the same
# path inside the container (bench client writes results there directly).
DTYPE_ARGS=()
if [[ "${DTYPE}" != "auto" ]]; then
DTYPE_ARGS=(--dtype "${DTYPE}")
fi
nohup docker run --rm \
--name "$CONTAINER_NAME" \
--gpus all \
--privileged \
--ipc=host \
--network host \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--entrypoint python3 \
-v /data:/data \
-v "${CACHE_DIR}:/root/.cache" \
-v "${RUNTIME_BASE}/tmp:/tmp" \
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
-e PYTHONUNBUFFERED=1 \
-e HF_HUB_OFFLINE=1 \
-e TRANSFORMERS_OFFLINE=1 \
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
"$DOCKER_IMAGE" \
-m sglang.launch_server \
--model-path "${MODEL_PATH}" \
--trust-remote-code \
--tp-size "${TP}" \
--mem-fraction-static "${MEM_FRACTION_STATIC}" \
--context-length "${CONTEXT_LENGTH}" \
--max-running-requests "${MAX_RUNNING_REQUESTS}" \
--host 0.0.0.0 \
--port "${PORT}" \
"${DTYPE_ARGS[@]}" \
> "$SERVER_LOG" 2>&1 &
PID=$!
echo "$PID" > "$PID_FILE"
log "PID: ${PID}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-1200}"
log "waiting for health on http://127.0.0.1:${PORT}/health (timeout=${HEALTH_TIMEOUT}s)"
if health_check 127.0.0.1 "$PORT" "$HEALTH_TIMEOUT"; then
log "sglang server is READY at http://127.0.0.1:${PORT}"
log "server log: ${SERVER_LOG} (container: ${CONTAINER_NAME})"
exit 0
else
log "ERROR: server not healthy after ${HEALTH_TIMEOUT}s"
log "----- last 200 lines of server log -----"
tail -200 "$SERVER_LOG" 2>/dev/null || docker logs --tail 200 "$CONTAINER_NAME" 2>&1 | tail -200
exit 1
fi
deploy_start \
"${DEPLOY_PROFILE:-pro6000/qwen3_235b_pro6000_sglang_tp8}" \
"${TP:-8}" "1" \
"${RUNTIME_BASE}/logs" \
"${PORT:-30010}" \
"$MODEL_PATH" \
"$CONTAINER_NAME"

23
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/stop_server.sh Executable file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Stop the Qwen3-235B-A22B SGLang server through the shared deployment layer.
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
log "stopping server for ${EXPERIMENT_NAME} via deploy profile"
deploy_stop \
"${DEPLOY_PROFILE:-pro6000/qwen3_235b_pro6000_sglang_tp8}" \
"${TP:-8}" "1" \
"${PORT:-30010}" \
"$MODEL_PATH" \
"$CONTAINER_NAME"

43
ops/README.md
View File

@ -80,7 +80,48 @@ experiments/<platform>/<experiment>/results/<run_id>/
跨机测试时可用 `--client docker --client-image lmsysorg/sglang:latest`
指定外部 client。
## 7. 常见问题
## 7. pro6000（RTX 6000D）平台说明
实验目录 `experiments/pro6000/`，部署 profile 见 `deploy/profiles/pro6000/`。
可测实验：
```bash
PYTHONPATH=src python3 -m sskj.bench list --platform pro6000
# dsv4_pro6000_sglang_tp16 / dsv4_pro6000_sglang_tp16_eagle
# dsv4_pro6000_sglang_tp_dp_matrix / dsv4_pro6000_vllm_tp_dp_matrix
# glm52_pro6000_sglang_multinode_tp16 / qwen3_235b_pro6000_sglang_tp8
```
单节点实验（dsv4/vllm tp_dp_matrix、qwen3，均 8x RTX 6000D 单机）：
```bash
PYTHONPATH=src python3 -m sskj.bench run \
--url http://<head-ip>:30031 \
--platform pro6000 \
--experiment dsv4_pro6000_sglang_tp_dp_matrix \
--profile smoke
```
多节点实验（tp16 / tp16_eagle / glm52，2 台机器 16x RTX 6000D）：
- head 节点 pro6000D.1（`10.101.0.11` / `174.1.51.5`，HTTP API 所在）
- worker 节点 pro6000D.3（`10.101.0.13` / `174.1.51.7`，纯计算）
- `--url` 固定指向 head 节点端口（tp16/eagle: 30000，glm52: 30031），
bench client 在 head 节点跑，worker 由部署层通过 ssh 管理：
```bash
PYTHONPATH=src python3 -m sskj.bench run \
--url http://10.101.0.11:30000 \
--platform pro6000 \
--experiment dsv4_pro6000_sglang_tp16 \
--profile smoke
```
vLLM 实验（`dsv4_pro6000_vllm_tp_dp_matrix`）的 bench client 使用 SGLang
镜像（vLLM 镜像不含 bench_serving），由 `DOCKER_CLIENT_IMAGE` 自动选择，
无需额外参数。
## 8. 常见问题
- 服务不健康：确认 `--url` 的端口可从测试机访问，服务已启动且 `/health` 可通。
- 数据集缺失：仓库 `datasets/` 下没有 ShareGPT 文件时，统一 CLI 会自动退回

68
scripts/common/deploy_cli.sh
View File

@ -19,6 +19,11 @@ deploy_start() {
local port="$5"
local model_path="$6"
local container="$7"
local node_rank="${8:-}"
local -a extra=()
if [[ -n "$node_rank" ]]; then
extra+=(--node-rank "$node_rank")
fi
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$profile")" \
--tp "$tp" \
@ -26,7 +31,8 @@ deploy_start() {
--log-dir "$log_dir" \
--port "$port" \
--model-path "$model_path" \
--container-name "$container"
--container-name "$container" \
"${extra[@]}"
}
deploy_render_args() {
@ -42,3 +48,63 @@ deploy_render_args() {
--port "$port" \
--model-path "$model_path"
}
# Multi-node convenience helpers: orchestrate all NODE_HOSTS from one host.
deploy_start_multinode() {
local profile="$1"
local tp="$2"
local dp="$3"
local log_dir="$4"
local port="$5"
local model_path="$6"
local container="$7"
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy start \
--profile "$(deploy_profile_abs "$profile")" \
--tp "$tp" \
--dp "$dp" \
--log-dir "$log_dir" \
--port "$port" \
--model-path "$model_path" \
--container-name "$container"
}
deploy_stop() {
local profile="$1"
local tp="$2"
local dp="$3"
local port="$4"
local model_path="$5"
local container="$6"
local node_rank="${7:-}"
local -a extra=()
if [[ -n "$node_rank" ]]; then
extra+=(--node-rank "$node_rank")
fi
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy stop \
--profile "$(deploy_profile_abs "$profile")" \
--tp "$tp" \
--dp "$dp" \
--port "$port" \
--model-path "$model_path" \
--container-name "$container" \
"${extra[@]}"
}
deploy_status() {
local profile="$1"
local tp="${2:-}"
local dp="${3:-}"
local port="${4:-}"
local model_path="${5:-}"
local container="${6:-}"
local -a cmd=(
--profile "$(deploy_profile_abs "$profile")"
)
[[ -n "$tp" ]] && cmd+=(--tp "$tp")
[[ -n "$dp" ]] && cmd+=(--dp "$dp")
[[ -n "$port" ]] && cmd+=(--port "$port")
[[ -n "$model_path" ]] && cmd+=(--model-path "$model_path")
[[ -n "$container" ]] && cmd+=(--container-name "$container")
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy status "${cmd[@]}"
}

66
src/sskj/common/env.py
View File

@ -3,7 +3,8 @@
The repository keeps deployment and platform configuration in shell `.env`
files so existing bash scripts can still source them. This parser understands
the scalar subset those files use: `KEY=value`, `export KEY=value`, quoted
values, comments, and `${VAR:-default}` substitution.
values, comments, and `${VAR:-default}` substitution. Nested defaults such as
`${A:-${B}/path}` are supported via balanced-brace scanning.
"""
from __future__ import annotations
@ -13,7 +14,7 @@ import re
from pathlib import Path
_ASSIGN_RE = re.compile(r"^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$")
_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
def _strip_quotes(value: str) -> str:
@ -42,20 +43,61 @@ def _strip_inline_comment(value: str) -> str:
return value
def _find_brace_end(value: str, start: int) -> int | None:
"""Index of the `}` closing the `${` at `start`, handling nested `${...}`."""
depth = 0
i = start
n = len(value)
while i < n:
if value.startswith("${", i):
depth += 1
i += 2
continue
if value[i] == "}":
depth -= 1
if depth == 0:
return i
i += 1
return None
def _split_default(inner: str) -> tuple[str, str | None]:
"""Split `NAME` or `NAME:-default` (default may itself contain `${...}`)."""
for sep in (":-", "-"):
idx = inner.find(sep)
if idx > 0 and _NAME_RE.match(inner[:idx]):
return inner[:idx], inner[idx + len(sep):]
if _NAME_RE.match(inner):
return inner, None
return inner, None
def _expand(value: str, env: dict[str, str], depth: int = 0) -> str:
if depth > 6:
return value
def repl(match: re.Match[str]) -> str:
name = match.group(1)
default = match.group(2)
if name in env and env[name] != "":
return env[name]
if default is not None:
return _expand(default, env, depth + 1)
return match.group(0)
return _VAR_RE.sub(repl, value)
out: list[str] = []
i = 0
n = len(value)
while i < n:
if value.startswith("${", i):
end = _find_brace_end(value, i)
if end is None:
out.append(value[i:])
break
inner = value[i + 2:end]
name, default = _split_default(inner)
if _NAME_RE.match(name) and name in env and env[name] != "":
out.append(env[name])
elif _NAME_RE.match(name) and default is not None:
out.append(_expand(default, env, depth + 1))
else:
out.append(value[i:end + 1])
i = end + 1
continue
out.append(value[i])
i += 1
return "".join(out)
def parse_env_file(

372
src/sskj/deploy/runtime.py
View File

@ -1,14 +1,33 @@
"""Docker/native server lifecycle for deployment profiles."""
"""Docker/native server lifecycle for deployment profiles.
Supports single-node and multi-node (NODE_HOSTS) docker deployments. For
multi-node profiles, start/stop/status distribute docker commands to each
node over ssh; health checks only target the head (rank 0) node.
"""
from __future__ import annotations
import base64
import re
import shlex
import subprocess
import time
from pathlib import Path
from sskj.bench.runner import split_env_assignments, wait_health
_NODE_RANK_RE = re.compile(r"\$\{NODE_RANK(?::-([^}]*))?\}")
_TEMPLATED_KEYS = (
"CONTAINER_NAME",
"LAUNCH_ARGS",
"DEVICE_VARS",
"ENGINE_ENV",
"DOCKER_FLAGS",
"VOLUMES",
"PATCH_MOUNTS",
"BOOTSTRAP",
)
def _log_dir(root: Path, log_dir: str | None) -> Path:
path = Path(log_dir) if log_dir else root / "deploy" / "logs"
@ -21,62 +40,218 @@ def _health_url(profile: dict[str, str]) -> str:
return f"http://{host}:{profile['PORT']}"
def _node_hosts(profile: dict[str, str]) -> list[str]:
return [h for h in shlex.split(profile.get("NODE_HOSTS", "")) if h]
def _ssh_host(profile: dict[str, str], host: str) -> str:
user = profile.get("NODE_SSH_USER", "root")
if not user or "@" in host:
return host
return f"{user}@{host}"
def _sub_node_rank(profile: dict[str, str], rank: int) -> dict[str, str]:
"""Return a profile copy with NODE_RANK rendered for the given rank."""
rendered = dict(profile)
container_name = str(rendered.get("CONTAINER_NAME", "server"))
has_template = _NODE_RANK_RE.search(container_name) is not None
def _sub(value: str | None) -> str:
if not value:
return value or ""
return _NODE_RANK_RE.sub(lambda m: m.group(1) or str(rank), str(value))
for key in _TEMPLATED_KEYS:
if key in rendered:
rendered[key] = _sub(rendered[key])
if not has_template:
rendered["CONTAINER_NAME"] = f"{container_name}_node{rank}"
rendered["NODE_RANK"] = str(rank)
return rendered
def _server_cmd(profile: dict[str, str]) -> str:
"""Launch command prefix (without LAUNCH_ARGS)."""
if profile.get("SERVER_CMD"):
return profile["SERVER_CMD"]
engine = (profile.get("ENGINE") or "sglang").lower()
python_bin = profile.get("CONTAINER_PYTHON", "python")
if engine == "vllm":
return f"{python_bin} -m vllm.entrypoints.openai.api_server"
return f"{python_bin} -m sglang.launch_server"
def _docker_run_cmd(profile: dict[str, str]) -> list[str]:
container = profile["CONTAINER_NAME"]
cmd = ["docker", "run", "-d", "--name", container]
if profile.get("DOCKER_FLAGS"):
cmd += shlex.split(profile["DOCKER_FLAGS"])
if profile.get("NETWORK_MODE") == "bridge" and profile.get("PORT_MAP") == "1":
cmd += ["-p", f"{profile['PORT']}:{profile['PORT']}"]
for volume in split_env_assignments(profile.get("VOLUMES", "") + " " + profile.get("PATCH_MOUNTS", "")):
cmd += ["-v", volume]
for assignment in split_env_assignments(
profile.get("DEVICE_VARS", "") + " " + profile.get("ENGINE_ENV", "")
):
cmd += ["-e", assignment]
cmd.append(profile["DOCKER_IMAGE"])
bootstrap = profile.get("BOOTSTRAP", "")
if bootstrap:
encoded = base64.b64encode(bootstrap.encode("utf-8")).decode("ascii")
cmd += ["bash", "-c", f"echo {encoded} | base64 -d | bash"]
else:
cmd += ["bash", "-c", f"{_server_cmd(profile)} {profile['LAUNCH_ARGS']}"]
return cmd
def _is_multinode(profile: dict[str, str], node_rank: str | None) -> bool:
return len(_node_hosts(profile)) > 1 and node_rank is None
def _local_rank(profile: dict[str, str]) -> int:
"""Rank that runs on this host (no ssh); defaults to 0 (head)."""
try:
return int(profile.get("LOCAL_NODE_RANK", "0"))
except ValueError:
return 0
def _node_target(profile: dict[str, str], rank: int) -> str | None:
"""ssh host for the rank, or None when the rank runs locally."""
if rank == _local_rank(profile):
return None
return _ssh_host(profile, _node_hosts(profile)[rank])
def _run_docker_cmd(
profile: dict[str, str],
cmd: list[str],
host: str | None,
dry_run: bool,
log_path: Path | None = None,
) -> int:
joined = shlex.join(cmd)
if log_path is not None:
log_path.write_text(joined + "\n", encoding="utf-8")
if dry_run:
print(joined)
return 0
if host:
remote = f"bash -c {shlex.quote(joined)}"
result = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", host, remote],
capture_output=True,
text=True,
)
if result.returncode != 0:
print(result.stdout, result.stderr)
return result.returncode
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(result.stdout, result.stderr)
return result.returncode
def _start_docker(
profile: dict[str, str],
root: Path,
dry_run: bool,
logs: Path,
) -> int:
if not profile.get("DOCKER_IMAGE"):
raise SystemExit("docker runtime requires DOCKER_IMAGE")
if not profile.get("CONTAINER_NAME"):
raise SystemExit("docker runtime requires CONTAINER_NAME")
hosts = _node_hosts(profile)
node_rank = profile.get("NODE_RANK")
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
if _is_multinode(profile, node_rank):
# Workers first (rank > 0), then head (rank 0), mirroring the legacy
# multi-node scripts so NCCL bootstrap finds the head up. The
# LOCAL_NODE_RANK rank runs locally (no ssh).
order = [r for r in range(len(hosts)) if r != 0] + [0]
for rank in order:
host = hosts[rank]
target = _node_target(profile, rank)
node_profile = _sub_node_rank(profile, rank)
cmd = _docker_run_cmd(node_profile)
if not dry_run:
if target:
subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker rm -f {shlex.quote(node_profile['CONTAINER_NAME'])}"],
capture_output=True,
check=False,
)
else:
subprocess.run(
["docker", "rm", "-f", node_profile["CONTAINER_NAME"]],
capture_output=True,
check=False,
)
print(f"starting container {node_profile['CONTAINER_NAME']} on {host} (rank {rank}) ...")
rc = _run_docker_cmd(
profile,
cmd,
target,
dry_run,
logs / f"{node_profile['CONTAINER_NAME']}.cmd.txt",
)
if rc != 0:
return rc
if rank != 0:
print(" sleeping 5s before head ...")
if not dry_run:
time.sleep(5)
head_url = f"http://{profile.get('HEALTH_HOST', hosts[0])}:{profile['PORT']}"
if dry_run:
print(f"[dry-run] health: {head_url}{profile.get('HEALTH_PATH', '/health')}")
return 0
if wait_health(head_url, profile.get("HEALTH_PATH", "/health"), health_wait):
print(f"server on {hosts[0]} is healthy")
return 0
print("ERROR: head failed health check; inspect per-node logs")
return 1
# Single-node or explicit --node-rank: run locally on this host.
if not dry_run:
subprocess.run(["docker", "rm", "-f", profile["CONTAINER_NAME"]], capture_output=True, check=False)
cmd = _docker_run_cmd(profile)
container = profile["CONTAINER_NAME"]
print(f"starting container {container} ...")
rc = _run_docker_cmd(profile, cmd, None, dry_run, logs / f"{container}.cmd.txt")
if rc != 0:
return rc
if node_rank is not None and node_rank != "0":
print(f"node rank {node_rank} started; skipping health check")
return 0
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
print(f"container {container} is healthy")
return 0
subprocess.run(["docker", "logs", "--tail", "100", container], check=False)
print(f"ERROR: container {container} failed health check")
return 1
def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: str | None = None) -> int:
validate_runtime(profile)
logs = _log_dir(root, log_dir)
runtime = profile["RUNTIME"]
container = profile.get("CONTAINER_NAME", "")
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
if runtime == "docker":
if not profile.get("DOCKER_IMAGE"):
raise SystemExit("docker runtime requires DOCKER_IMAGE")
if not container:
raise SystemExit("docker runtime requires CONTAINER_NAME")
if not dry_run:
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
cmd = ["docker", "run", "-d", "--name", container]
if profile.get("DOCKER_FLAGS"):
cmd += shlex.split(profile["DOCKER_FLAGS"])
if profile.get("NETWORK_MODE") == "bridge" and profile.get("PORT_MAP") == "1":
cmd += ["-p", f"{profile['PORT']}:{profile['PORT']}"]
for volume in split_env_assignments(profile.get("VOLUMES", "") + " " + profile.get("PATCH_MOUNTS", "")):
cmd += ["-v", volume]
for assignment in split_env_assignments(
profile.get("DEVICE_VARS", "") + " " + profile.get("ENGINE_ENV", "")
):
cmd += ["-e", assignment]
cmd.append(profile["DOCKER_IMAGE"])
bootstrap = profile.get("BOOTSTRAP", "")
if bootstrap:
encoded = base64.b64encode(bootstrap.encode("utf-8")).decode("ascii")
cmd += ["bash", "-c", f"echo {encoded} | base64 -d | bash"]
else:
python_bin = profile.get("CONTAINER_PYTHON", "python")
cmd += ["bash", "-c", f"{python_bin} -m sglang.launch_server {profile['LAUNCH_ARGS']}"]
(logs / f"{container}.cmd.txt").write_text(shlex.join(cmd) + "\n", encoding="utf-8")
if dry_run:
print(shlex.join(cmd))
return 0
print(f"starting container {container} ...")
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print(result.stdout, result.stderr)
return result.returncode
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
print(f"container {container} is healthy")
return 0
subprocess.run(["docker", "logs", "--tail", "100", container], check=False)
print(f"ERROR: container {container} failed health check")
return 1
if profile["RUNTIME"] == "docker":
return _start_docker(profile, root, dry_run, logs)
# native runtime
python_bin = profile.get("PYTHON_BIN") or profile.get("CONTAINER_PYTHON") or "python3"
launch_args = shlex.split(profile.get("LAUNCH_ARGS", ""))
cmd = [python_bin, "-m", "sglang.launch_server", *launch_args]
if profile.get("SERVER_CMD"):
cmd = shlex.split(profile["SERVER_CMD"]) + shlex.split(profile.get("LAUNCH_ARGS", ""))
else:
engine = (profile.get("ENGINE") or "sglang").lower()
module = "vllm.entrypoints.openai.api_server" if engine == "vllm" else "sglang.launch_server"
cmd = [python_bin, "-m", module, *shlex.split(profile.get("LAUNCH_ARGS", ""))]
log_path = logs / f"{profile.get('MODEL_NAME', 'server')}.log"
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
if dry_run:
@ -86,6 +261,7 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
proc = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT)
pid_file.write_text(str(proc.pid), encoding="utf-8")
print(f"started native server pid={proc.pid} log={log_path}")
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
print("native server is healthy")
return 0
@ -93,15 +269,45 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
return 1
def _docker_stop(profile: dict[str, str], dry_run: bool = False) -> int:
node_rank = profile.get("NODE_RANK")
if _is_multinode(profile, node_rank):
for rank, host in enumerate(_node_hosts(profile)):
node_profile = _sub_node_rank(profile, rank)
name = node_profile["CONTAINER_NAME"]
target = _node_target(profile, rank)
if dry_run:
if target:
print(f"[dry-run] ssh {target} docker rm -f {name}")
else:
print(f"[dry-run] docker rm -f {name} (local)")
continue
if target:
subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker rm -f {shlex.quote(name)}"],
capture_output=True,
check=False,
)
print(f"container {name} removed on {host}")
else:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
print(f"container {name} removed (local)")
return 0
container = profile.get("CONTAINER_NAME")
if not container:
raise SystemExit("docker runtime requires CONTAINER_NAME")
if dry_run:
print(f"[dry-run] docker rm -f {container}")
return 0
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
print(f"container {container} removed")
return 0
def stop(profile: dict[str, str], root: Path) -> int:
validate_runtime(profile)
if profile["RUNTIME"] == "docker":
container = profile.get("CONTAINER_NAME")
if not container:
raise SystemExit("docker runtime requires CONTAINER_NAME")
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
print(f"container {container} removed")
return 0
return _docker_stop(profile)
logs = _log_dir(root, None)
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
if pid_file.exists():
@ -113,18 +319,52 @@ def stop(profile: dict[str, str], root: Path) -> int:
return 0
def _docker_status(profile: dict[str, str]) -> int:
node_rank = profile.get("NODE_RANK")
if _is_multinode(profile, node_rank):
all_ok = 0
for rank, host in enumerate(_node_hosts(profile)):
node_profile = _sub_node_rank(profile, rank)
name = node_profile["CONTAINER_NAME"]
target = _node_target(profile, rank)
if target:
result = subprocess.run(
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker inspect -f '{{{{.State.Status}}}}' {shlex.quote(name)}"],
capture_output=True,
text=True,
check=False,
)
state = result.stdout.strip() or "missing"
print(f"{host}: {name} -> {state}")
if result.returncode != 0 or state != "running":
all_ok = 1
else:
result = subprocess.run(
["docker", "inspect", "-f", "{{.State.Status}}", name],
capture_output=True,
text=True,
check=False,
)
state = result.stdout.strip() or "missing"
print(f"{host}: {name} -> {state}")
if result.returncode != 0 or state != "running":
all_ok = 1
return all_ok
container = profile.get("CONTAINER_NAME", "")
result = subprocess.run(
["docker", "inspect", "-f", "{{.State.Status}}", container],
capture_output=True,
text=True,
check=False,
)
print(result.stdout.strip() or "missing")
return 0 if result.returncode == 0 else 1
def status(profile: dict[str, str], root: Path) -> int:
validate_runtime(profile)
if profile["RUNTIME"] == "docker":
container = profile.get("CONTAINER_NAME", "")
result = subprocess.run(
["docker", "inspect", "-f", "{{.State.Status}}", container],
capture_output=True,
text=True,
check=False,
)
print(result.stdout.strip() or "missing")
return 0 if result.returncode == 0 else 1
return _docker_status(profile)
logs = _log_dir(root, None)
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
if not pid_file.exists():