feat(pro6000): 部署/测试解耦 - deploy 层支持多节点与 vLLM，新增 6 个 profile
- sskj.deploy runtime 支持 NODE_HOSTS 多节点编排(ssh 分发/本地 rank/LOCAL_NODE_RANK)
与 ENGINE=vllm 启动(SERVER_CMD)，容器名按 rank 自动唯一
- scripts/common/deploy_cli.sh 新增 deploy_stop/status/multinode helper 与 node-rank 透传
- src/sskj/common/env.py 修复嵌套 ${VAR:-${OTHER}/path} 展开(平衡花括号扫描)
- deploy/profiles/pro6000/ 新增 6 个 profile: tp16/tp16_eagle/glm52(多节点)、
sglang/vllm tp_dp_matrix、qwen3(单节点)
- 6 个实验 start/stop 脚本改为 deploy 薄包装，run_bench/adaptive 的 server 启停走
deploy_render_args/deploy_start/deploy_stop，tp16 新增 matrix.json
- 首次入库 glm52_pro6000_sglang_multinode_tp16 实验目录；ops/README.md 补 pro6000 章节
- 实测通过: 单节点 dsv4 sglang/vllm 链路 + tp16 双节点启动/bench/清理
This commit is contained in:
Zhiyi Hong
parent
3761d75b00
commit
9acf9fdfdb
10
.gitignore
vendored
10
.gitignore
vendored
@ -76,14 +76,21 @@ build/
|
|||||||
sglang_sm120_cache/
|
sglang_sm120_cache/
|
||||||
vllm_sm120_cache/
|
vllm_sm120_cache/
|
||||||
*_sm120_cache/
|
*_sm120_cache/
|
||||||
|
sglang_nightly_cu13_cache/
|
||||||
|
|
||||||
# 一次性 bench 输出
|
# 一次性 bench 输出
|
||||||
bench-output/
|
bench-output/
|
||||||
|
|
||||||
|
# 单数 dataset/（可重新下载，与 datasets/ 同源）
|
||||||
|
dataset/
|
||||||
|
|
||||||
# 逐请求原始日志（体积大；汇总见 results.json / report.md）
|
# 逐请求原始日志（体积大；汇总见 results.json / report.md）
|
||||||
experiments/**/raw_outputs/
|
experiments/**/raw_outputs/
|
||||||
**/dummy_sharegpt.json
|
**/dummy_sharegpt.json
|
||||||
|
|
||||||
|
# pro6000 统一 bench CLI 的一次性测试输出（ops_ 前缀 run_id）
|
||||||
|
experiments/pro6000/*/results/ops_*/
|
||||||
|
|
||||||
# Virtual environments (keep only docs under envs/)
|
# Virtual environments (keep only docs under envs/)
|
||||||
envs/*
|
envs/*
|
||||||
!envs/README.md
|
!envs/README.md
|
||||||
@ -96,3 +103,6 @@ skills-lock.json
|
|||||||
|
|
||||||
# 临时备份文件
|
# 临时备份文件
|
||||||
*.bak.glm52orig
|
*.bak.glm52orig
|
||||||
|
*.bak
|
||||||
|
*.bak.*
|
||||||
|
*.tmp
|
||||||
|
|||||||
34
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16.env
Normal file
34
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16.env
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
# DeepSeek-V4-Pro SGLang multi-node TP=16 EP=2 deployment profile (2x RTX 6000D).
|
||||||
|
# Nodes: 10.101.0.11 (rank 0) + 10.101.0.13 (rank 1), 16x RTX 6000D (85GB).
|
||||||
|
# Key: SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2 -> TP=16 works with FP8 block=128.
|
||||||
|
# Model-team only. Ops only run `python -m sskj.bench` against the served URL.
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=dsv4_pro6000_sglang_tp16
|
||||||
|
MODEL_NAME=DeepSeek-V4-Pro
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
|
||||||
|
MODEL_PATH=/data/hf_models/DeepSeek-V4-Pro
|
||||||
|
SERVED_MODEL_NAME=default
|
||||||
|
PORT=30000
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_HOST=10.101.0.11
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
|
||||||
|
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
|
||||||
|
NNODES=2
|
||||||
|
NODE_HOSTS="10.101.0.11 10.101.0.13"
|
||||||
|
NODE_SSH_USER=root
|
||||||
|
MASTER_IP=10.101.0.11
|
||||||
|
DIST_PORT=20000
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="NCCL_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN SGLANG_SHARED_EXPERT_TP1=1 PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --tp-size ${TP} --ep-size 2 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --host 0.0.0.0 --port ${PORT} --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256"
|
||||||
32
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16_eagle.env
Normal file
32
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp16_eagle.env
Normal file
@ -0,0 +1,32 @@
|
|||||||
|
# DeepSeek-V4-Pro SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding
|
||||||
|
# (2x RTX 6000D). Same topology as dsv4_pro6000_sglang_tp16, plus EAGLE flags.
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=dsv4_pro6000_sglang_tp16_eagle
|
||||||
|
MODEL_NAME=DeepSeek-V4-Pro
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
|
||||||
|
MODEL_PATH=/data/hf_models/DeepSeek-V4-Pro
|
||||||
|
SERVED_MODEL_NAME=default
|
||||||
|
PORT=30000
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_HOST=10.101.0.11
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
|
||||||
|
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
|
||||||
|
NNODES=2
|
||||||
|
NODE_HOSTS="10.101.0.11 10.101.0.13"
|
||||||
|
NODE_SSH_USER=root
|
||||||
|
MASTER_IP=10.101.0.11
|
||||||
|
DIST_PORT=20000
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="NCCL_SOCKET_IFNAME=eth0 NCCL_DEBUG=WARN SGLANG_SHARED_EXPERT_TP1=1 PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --tp-size ${TP} --ep-size 2 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --host 0.0.0.0 --port ${PORT} --mem-fraction-static 0.9 --cuda-graph-max-bs-decode 64 --max-running-requests 256 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4"
|
||||||
24
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp_dp_matrix.env
Normal file
24
deploy/profiles/pro6000/dsv4_pro6000_sglang_tp_dp_matrix.env
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# DeepSeek-V4-Flash SGLang TPxDP matrix profile on a single RTX 6000D node (8 GPUs).
|
||||||
|
# TP=2/DP=4 is excluded because Marlin weight loading OOMs on this machine.
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=dsv4_pro6000_sglang_tp_dp_matrix
|
||||||
|
MODEL_NAME=DeepSeek-V4-Flash
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_sglang_tp${TP}_dp${DP}
|
||||||
|
MODEL_PATH=/data/6000D/DeepSeek-V4-Flash
|
||||||
|
SERVED_MODEL_NAME=deepseek-v4-flash
|
||||||
|
PORT=30031
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --moe-runner-backend auto --mem-fraction-static 0.9 --context-length 131072 --max-running-requests 64 --host 0.0.0.0 --port ${PORT}"
|
||||||
24
deploy/profiles/pro6000/dsv4_pro6000_vllm_tp_dp_matrix.env
Normal file
24
deploy/profiles/pro6000/dsv4_pro6000_vllm_tp_dp_matrix.env
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# DeepSeek-V4-Flash vLLM TPxDP matrix profile on a single RTX 6000D node (8 GPUs).
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=dsv4_pro6000_vllm_tp_dp_matrix
|
||||||
|
MODEL_NAME=DeepSeek-V4-Flash
|
||||||
|
ENGINE=vllm
|
||||||
|
SERVER_CMD=vllm serve
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=vllm-sm120-dsv4:0.25.1-fi0.6.14
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_vllm_tp${TP}_dp${DP}
|
||||||
|
MODEL_PATH=/data/6000D/DeepSeek-V4-Flash
|
||||||
|
SERVED_MODEL_NAME=deepseek-v4-flash
|
||||||
|
PORT=30030
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model ${MODEL_PATH} --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size ${TP} --gpu-memory-utilization 0.9 --max-model-len 131072 --max-num-seqs 128 --host 0.0.0.0 --port ${PORT}"
|
||||||
|
DP_FLAG="--data-parallel-size ${DP}"
|
||||||
37
deploy/profiles/pro6000/glm52_pro6000_sglang_tp16.env
Normal file
37
deploy/profiles/pro6000/glm52_pro6000_sglang_tp16.env
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
# GLM-5.2-FP8 SGLang multi-node TP=16 deployment profile (2x RTX 6000D).
|
||||||
|
# node0 = 174.1.51.5 (pro6000D.1, HTTP API), node1 = 174.1.51.7 (pro6000D.3,
|
||||||
|
# compute-only). Model ~700GB fp8 so 2-node TP=16 is mandatory.
|
||||||
|
# Requires the tilelang/dsa patch mounts below (experiment-dir sources).
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=glm52_pro6000_sglang_multinode_tp16
|
||||||
|
MODEL_NAME=GLM-5.2-FP8
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_sglang_tp${TP}_dp${DP}_node${NODE_RANK}
|
||||||
|
MODEL_PATH=/data/hf_models/GLM-5.2-FP8
|
||||||
|
SERVED_MODEL_NAME=GLM-5.2-FP8
|
||||||
|
PORT=30031
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_HOST=174.1.51.5
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
TP=16
|
||||||
|
DP=1
|
||||||
|
|
||||||
|
# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
|
||||||
|
NNODES=2
|
||||||
|
NODE_HOSTS="174.1.51.5 174.1.51.7"
|
||||||
|
NODE_SSH_USER=root
|
||||||
|
MASTER_IP=174.1.51.5
|
||||||
|
DIST_PORT=50000
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:256 NCCL_IB_HCA=mlx5_0,mlx5_3 NCCL_MIN_NCHANNELS=8 NCCL_IB_QPS_PER_CONNECTION=4 NCCL_NET_GDR_LEVEL=PHB NCCL_SOCKET_IFNAME=eth1 NCCL_IB_RETRY_CNT=14 NCCL_IB_TIMEOUT=16"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro ${ROOT_DIR}/sglang_nightly_cu13_cache:/root/.cache"
|
||||||
|
PATCH_MOUNTS="${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/tilelang_kernel_sm120.py:/sgl-workspace/sglang/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_backend.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa_backend.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_indexer.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa/dsa_indexer.py:ro"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --dp-size ${DP} --quantization fp8 --kv-cache-dtype bfloat16 --moe-runner-backend auto --mem-fraction-static 0.80 --context-length 131072 --max-running-requests 64 --cuda-graph-backend-decode disabled --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --host 0.0.0.0 --port ${PORT} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --nnodes ${NNODES} --node-rank ${NODE_RANK}"
|
||||||
26
deploy/profiles/pro6000/qwen3_235b_pro6000_sglang_tp8.env
Normal file
26
deploy/profiles/pro6000/qwen3_235b_pro6000_sglang_tp8.env
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# Qwen3-235B-A22B SGLang TP=8 profile on a single RTX 6000D node (8 GPUs).
|
||||||
|
# NVIDIA counterpart of p800/qwen3_235b_p800_sglang_tp8 (same workload for
|
||||||
|
# comparability: isl=2048 osl=2048 c=16).
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=qwen3_235b_pro6000_sglang_tp8
|
||||||
|
MODEL_NAME=Qwen3-235B-A22B
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1
|
||||||
|
CONTAINER_NAME=qwen3_235b_pro6000_sglang_tp8
|
||||||
|
MODEL_PATH=/data/6000D/Qwen3-235B-A22B
|
||||||
|
SERVED_MODEL_NAME=/data/6000D/Qwen3-235B-A22B
|
||||||
|
PORT=30010
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_WAIT_S=1800
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
TP=8
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
|
||||||
|
VOLUMES="/data:/data"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --mem-fraction-static 0.9 --context-length 8192 --max-running-requests 16 --host 0.0.0.0 --port ${PORT}"
|
||||||
128
experiments/pro6000/dsv4_pro6000_sglang_tp16/matrix.json
Normal file
128
experiments/pro6000/dsv4_pro6000_sglang_tp16/matrix.json
Normal file
@ -0,0 +1,128 @@
|
|||||||
|
{
|
||||||
|
"comment": "ISL/OSL matrix for dsv4_pro6000_sglang_tp16 (multi-node TP16 EP2). Y=must test, P=optional, N=skip. Extracted from run_batch.sh legacy inline matrix; concurrency per ISL mirrors get_conc_for().",
|
||||||
|
"mode": "Y",
|
||||||
|
"matrix": {
|
||||||
|
"1024": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "P",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "P",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "P",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"concurrency": {
|
||||||
|
"1024": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 128
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 128
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 32
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 32
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 8
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 8
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -152,9 +152,9 @@ check_and_recover_server() {
|
|||||||
fi
|
fi
|
||||||
log "WARN: Server health failed. Checking..."
|
log "WARN: Server health failed. Checking..."
|
||||||
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
|
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
|
||||||
"docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "")
|
"docker ps --filter name=${EXPERIMENT}_node0 --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
|
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
|
||||||
"docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "")
|
"docker ps --filter name=${EXPERIMENT}_node1 --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
if [[ -z "$hok" || -z "$wok" ]]; then
|
if [[ -z "$hok" || -z "$wok" ]]; then
|
||||||
log "ERROR: Containers died. Restarting server..."
|
log "ERROR: Containers died. Restarting server..."
|
||||||
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }
|
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }
|
||||||
|
|||||||
@ -1,84 +1,40 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# ============================================================
|
# Start SGLang multi-node TP=16 EP=2 across 2 nodes through the shared
|
||||||
# Start SGLang multi-node TP=16 EP=2 across 2 nodes.
|
# deployment layer (`python -m sskj.deploy start`, NODE_HOSTS orchestration).
|
||||||
# Key: SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2
|
# Usage: start_sglang_multinode.sh [--dry-run]
|
||||||
# Usage: ./start_sglang_multinode.sh [--dry-run]
|
|
||||||
# ============================================================
|
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs"
|
||||||
|
|
||||||
DRY_RUN="${DRY_RUN:-0}"
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
HEAD_CONTAINER="${EXPERIMENT}_head"
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16}"
|
||||||
WORKER_CONTAINER="${EXPERIMENT}_worker"
|
|
||||||
|
|
||||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
|
|
||||||
build_docker_cmd() {
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
local node_rank="$1" container_name="$2"
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
|
||||||
cat <<DOCKER_CMD
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
docker rm -f ${container_name} 2>/dev/null;
|
--tp "$TP_SIZE" --dp "1" \
|
||||||
docker run -d --name ${container_name} \\
|
--port "${SGLANG_PORT:-30000}" \
|
||||||
--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\
|
--model-path "$MODEL_PATH" \
|
||||||
-v ${MODEL_PATH}:${MODEL_PATH}:ro \\
|
--log-dir "${RUNTIME_BASE}/logs" \
|
||||||
-e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\
|
--dry-run
|
||||||
-e NCCL_DEBUG=${NCCL_DEBUG} \\
|
else
|
||||||
-e SGLANG_SHARED_EXPERT_TP1=1 \\
|
deploy_start_multinode \
|
||||||
${DOCKER_IMAGE} \\
|
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
|
||||||
python3 -m sglang.launch_server \\
|
"${RUNTIME_BASE}/logs" \
|
||||||
--model-path ${MODEL_PATH} \\
|
"${SGLANG_PORT:-30000}" \
|
||||||
--tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\
|
"$MODEL_PATH" \
|
||||||
--node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\
|
"$EXPERIMENT"
|
||||||
--trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\
|
fi
|
||||||
--mem-fraction-static ${MEM_FRACTION_STATIC} \\
|
|
||||||
--cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\
|
|
||||||
--max-running-requests ${MAX_RUNNING_REQUESTS}
|
|
||||||
DOCKER_CMD
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} ==="
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == "0" ]]; then
|
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true
|
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true
|
|
||||||
sleep 2
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..."
|
|
||||||
local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")"
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null
|
|
||||||
log "Worker started."
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..."
|
|
||||||
cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")"
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null
|
|
||||||
log "Head started."
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Health check
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi
|
|
||||||
log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..."
|
|
||||||
for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do
|
|
||||||
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
|
||||||
log "Server healthy! (${i}s)"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" \
|
|
||||||
"docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
|
||||||
local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" \
|
|
||||||
"docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
|
||||||
if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi
|
|
||||||
if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi
|
|
||||||
if (( i % 12 == 0 )); then log " waiting... ${i}s"; fi
|
|
||||||
sleep "${HEALTH_CHECK_INTERVAL_S}"
|
|
||||||
done
|
|
||||||
log "ERROR: Timeout"; return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
|
|||||||
@ -1,14 +1,36 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Stop SGLang multi-node server and clean up.
|
# Stop SGLang multi-node server (both nodes) through the shared deployment layer.
|
||||||
set -euo pipefail
|
# Usage: stop_sglang_multinode.sh [--dry-run]
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
log "Stopping ${EXPERIMENT} ..."
|
log "=== Stopping SGLang multi-node via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" \
|
|
||||||
"docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'"
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" \
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
|
||||||
"docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'"
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
|
--tp "$TP_SIZE" --dp "1" \
|
||||||
|
--port "${SGLANG_PORT:-30000}" \
|
||||||
|
--model-path "$MODEL_PATH" \
|
||||||
|
--container-name "$EXPERIMENT" \
|
||||||
|
--dry-run
|
||||||
|
else
|
||||||
|
deploy_stop \
|
||||||
|
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
|
||||||
|
"${SGLANG_PORT:-30000}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$EXPERIMENT"
|
||||||
|
fi
|
||||||
log "Done."
|
log "Done."
|
||||||
|
|||||||
@ -152,9 +152,9 @@ check_and_recover_server() {
|
|||||||
fi
|
fi
|
||||||
log "WARN: Server health failed. Checking..."
|
log "WARN: Server health failed. Checking..."
|
||||||
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
|
local hok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${HEAD_NODE}" \
|
||||||
"docker ps --filter name=${EXPERIMENT}_head --format '{{.Names}}' 2>/dev/null" || echo "")
|
"docker ps --filter name=${EXPERIMENT}_node0 --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
|
local wok=$(ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 "${WORKER_NODE}" \
|
||||||
"docker ps --filter name=${EXPERIMENT}_worker --format '{{.Names}}' 2>/dev/null" || echo "")
|
"docker ps --filter name=${EXPERIMENT}_node1 --format '{{.Names}}' 2>/dev/null" || echo "")
|
||||||
if [[ -z "$hok" || -z "$wok" ]]; then
|
if [[ -z "$hok" || -z "$wok" ]]; then
|
||||||
log "ERROR: Containers died. Restarting server..."
|
log "ERROR: Containers died. Restarting server..."
|
||||||
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }
|
bash "${SCRIPT_DIR}/start_sglang_multinode.sh" || { log "FATAL: Restart failed"; return 1; }
|
||||||
|
|||||||
@ -1,82 +1,40 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# ============================================================
|
# Start SGLang multi-node TP=16 EP=2 across 2 nodes through the shared
|
||||||
# Start SGLang multi-node TP=16 EP=2 + EAGLE speculative decoding
|
# deployment layer (`python -m sskj.deploy start`, NODE_HOSTS orchestration).
|
||||||
# Usage: ./start_sglang_multinode.sh [--dry-run]
|
# Usage: start_sglang_multinode.sh [--dry-run]
|
||||||
# ============================================================
|
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs"
|
||||||
|
|
||||||
DRY_RUN="${DRY_RUN:-0}"
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
HEAD_CONTAINER="${EXPERIMENT}_head"
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16_eagle}"
|
||||||
WORKER_CONTAINER="${EXPERIMENT}_worker"
|
|
||||||
|
|
||||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
log "=== SGLang multi-node TP=${TP_SIZE} EP=${EP_SIZE} via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
|
|
||||||
build_docker_cmd() {
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
local node_rank="$1" container_name="$2"
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
|
||||||
cat <<DOCKER_CMD
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
docker rm -f ${container_name} 2>/dev/null;
|
--tp "$TP_SIZE" --dp "1" \
|
||||||
docker run -d --name ${container_name} \\
|
--port "${SGLANG_PORT:-30000}" \
|
||||||
--gpus all --network host --ipc=host --ulimit memlock=-1 --shm-size=20g \\
|
--model-path "$MODEL_PATH" \
|
||||||
-v ${MODEL_PATH}:${MODEL_PATH}:ro \\
|
--log-dir "${RUNTIME_BASE}/logs" \
|
||||||
-e NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME} \\
|
--dry-run
|
||||||
-e NCCL_DEBUG=${NCCL_DEBUG} \\
|
else
|
||||||
-e SGLANG_SHARED_EXPERT_TP1=1 \\
|
deploy_start_multinode \
|
||||||
${DOCKER_IMAGE} \\
|
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
|
||||||
python3 -m sglang.launch_server \\
|
"${RUNTIME_BASE}/logs" \
|
||||||
--model-path ${MODEL_PATH} \\
|
"${SGLANG_PORT:-30000}" \
|
||||||
--tp-size ${TP_SIZE} --ep-size ${EP_SIZE} --nnodes ${NNODES} \\
|
"$MODEL_PATH" \
|
||||||
--node-rank ${node_rank} --dist-init-addr ${HEAD_IP}:${DIST_INIT_PORT} \\
|
"$EXPERIMENT"
|
||||||
--trust-remote-code --host 0.0.0.0 --port ${SGLANG_PORT} \\
|
fi
|
||||||
--mem-fraction-static ${MEM_FRACTION_STATIC} \\
|
|
||||||
--cuda-graph-max-bs-decode ${CUDA_GRAPH_MAX_BS_DECODE} \\
|
|
||||||
--max-running-requests ${MAX_RUNNING_REQUESTS} \\
|
|
||||||
--speculative-algorithm ${SPECULATIVE_ALGORITHM} \\
|
|
||||||
--speculative-num-steps ${SPECULATIVE_NUM_STEPS} \\
|
|
||||||
--speculative-eagle-topk ${SPECULATIVE_EAGLE_TOPK} \\
|
|
||||||
--speculative-num-draft-tokens ${SPECULATIVE_NUM_DRAFT_TOKENS}
|
|
||||||
DOCKER_CMD
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
log "=== SGLang TP=${TP_SIZE} EP=${EP_SIZE} + EAGLE ==="
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == "0" ]]; then
|
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${HEAD_CONTAINER} 2>/dev/null" || true
|
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${WORKER_CONTAINER} 2>/dev/null" || true
|
|
||||||
sleep 2
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Starting WORKER (${WORKER_NODE}, node_rank=1) ..."
|
|
||||||
local cmd; cmd="$(build_docker_cmd 1 "${WORKER_CONTAINER}")"
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "bash -c '${cmd}'" >/dev/null
|
|
||||||
log "Worker started."
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
|
|
||||||
log "Starting HEAD (${HEAD_NODE}, node_rank=0) ..."
|
|
||||||
cmd="$(build_docker_cmd 0 "${HEAD_CONTAINER}")"
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] $cmd"; else
|
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "bash -c '${cmd}'" >/dev/null
|
|
||||||
log "Head started."
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == "1" ]]; then log "[DRY] Skip health check"; return 0; fi
|
|
||||||
log "Waiting for health on ${HEAD_IP}:${SGLANG_PORT} ..."
|
|
||||||
for (( i=1; i<=${HEALTH_CHECK_RETRIES}; i++ )); do
|
|
||||||
if curl --fail --silent --max-time 5 "http://${HEAD_IP}:${SGLANG_PORT}/health" >/dev/null 2>&1; then
|
|
||||||
log "Server healthy! (${i}s)"; return 0
|
|
||||||
fi
|
|
||||||
local hok=$(ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker ps --filter name=${HEAD_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
|
||||||
local wok=$(ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker ps --filter name=${WORKER_CONTAINER} --format '{{.Names}}' 2>/dev/null" || true)
|
|
||||||
if [[ -z "$hok" ]]; then log "ERROR: Head died"; ssh "${HEAD_NODE}" "docker logs --tail 30 ${HEAD_CONTAINER}" || true; return 1; fi
|
|
||||||
if [[ -z "$wok" ]]; then log "ERROR: Worker died"; ssh "${WORKER_NODE}" "docker logs --tail 30 ${WORKER_CONTAINER}" || true; return 1; fi
|
|
||||||
sleep "${HEALTH_CHECK_INTERVAL_S}"
|
|
||||||
done
|
|
||||||
log "ERROR: Timeout"; return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
|
|||||||
@ -1,9 +1,36 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -euo pipefail
|
# Stop SGLang multi-node server (both nodes) through the shared deployment layer.
|
||||||
|
# Usage: stop_sglang_multinode.sh [--dry-run]
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
log() { echo "[$(date '+%H:%M:%S')] $*"; }
|
# shellcheck source=/dev/null
|
||||||
log "Stopping ${EXPERIMENT} ..."
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
ssh -o StrictHostKeyChecking=no "${HEAD_NODE}" "docker rm -f ${EXPERIMENT}_head 2>/dev/null && echo 'Head stopped' || echo 'Head already stopped'"
|
# shellcheck source=/dev/null
|
||||||
ssh -o StrictHostKeyChecking=no "${WORKER_NODE}" "docker rm -f ${EXPERIMENT}_worker 2>/dev/null && echo 'Worker stopped' || echo 'Worker already stopped'"
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp16_eagle}"
|
||||||
|
|
||||||
|
log "=== Stopping SGLang multi-node via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
|
||||||
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
|
--tp "$TP_SIZE" --dp "1" \
|
||||||
|
--port "${SGLANG_PORT:-30000}" \
|
||||||
|
--model-path "$MODEL_PATH" \
|
||||||
|
--container-name "$EXPERIMENT" \
|
||||||
|
--dry-run
|
||||||
|
else
|
||||||
|
deploy_stop \
|
||||||
|
"$DEPLOY_PROFILE" "$TP_SIZE" "1" \
|
||||||
|
"${SGLANG_PORT:-30000}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$EXPERIMENT"
|
||||||
|
fi
|
||||||
log "Done."
|
log "Done."
|
||||||
|
|||||||
@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
|
|||||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}"
|
||||||
|
|
||||||
ENGINE="sglang"
|
ENGINE="sglang"
|
||||||
ENGINE_PORT="$SGLANG_PORT"
|
ENGINE_PORT="$SGLANG_PORT"
|
||||||
@ -37,29 +41,9 @@ engine_is_healthy() {
|
|||||||
engine_stop_server() {
|
engine_stop_server() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
|
||||||
|
|
||||||
if [[ -f "$pid_file" ]]; then
|
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
|
||||||
local pid
|
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
elif kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
fi
|
|
||||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
ACTIVE_ENGINE_SERVER_LOG=""
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
sleep 2
|
sleep 2
|
||||||
}
|
}
|
||||||
@ -67,21 +51,7 @@ engine_stop_server() {
|
|||||||
engine_build_server_args() {
|
engine_build_server_args() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local -a args=(
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--tp-size "$tp"
|
|
||||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
|
||||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
|
||||||
--context-length "$CONTEXT_LENGTH"
|
|
||||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$ENGINE_PORT"
|
|
||||||
)
|
|
||||||
if (( dp > 1 )); then
|
|
||||||
args+=(--dp-size "$dp")
|
|
||||||
fi
|
|
||||||
printf '%q ' "${args[@]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
engine_start_server() {
|
engine_start_server() {
|
||||||
|
|||||||
@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
|
|||||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}"
|
||||||
|
|
||||||
ENGINE="sglang"
|
ENGINE="sglang"
|
||||||
ENGINE_PORT="$SGLANG_PORT"
|
ENGINE_PORT="$SGLANG_PORT"
|
||||||
@ -37,29 +41,9 @@ engine_is_healthy() {
|
|||||||
engine_stop_server() {
|
engine_stop_server() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
|
||||||
|
|
||||||
if [[ -f "$pid_file" ]]; then
|
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
|
||||||
local pid
|
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
elif kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
fi
|
|
||||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
ACTIVE_ENGINE_SERVER_LOG=""
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
sleep 2
|
sleep 2
|
||||||
}
|
}
|
||||||
@ -67,21 +51,7 @@ engine_stop_server() {
|
|||||||
engine_build_server_args() {
|
engine_build_server_args() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local -a args=(
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
python3 -m sglang.launch_server --model-path "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--tp-size "$tp"
|
|
||||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
|
||||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
|
||||||
--context-length "$CONTEXT_LENGTH"
|
|
||||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$ENGINE_PORT"
|
|
||||||
)
|
|
||||||
if (( dp > 1 )); then
|
|
||||||
args+=(--dp-size "$dp")
|
|
||||||
fi
|
|
||||||
printf '%q ' "${args[@]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
engine_start_server() {
|
engine_start_server() {
|
||||||
@ -181,6 +151,8 @@ export -f engine_run_bench
|
|||||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
|
|
||||||
export SEARCH_START_CONCURRENCY=16
|
export SEARCH_START_CONCURRENCY=16
|
||||||
export SEARCH_ADDEND=16
|
export SEARCH_ADDEND=16
|
||||||
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
@ -189,5 +161,3 @@ export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
|||||||
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
# shapes in this TP/DP group. Zero disables this rule.
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
|
|
||||||
adaptive_main "$@"
|
|
||||||
|
|||||||
@ -11,6 +11,8 @@ source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
|||||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||||
RESULT_BASE="${SCRIPT_DIR}/results"
|
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||||
@ -51,33 +53,8 @@ stop_server() {
|
|||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
|
|
||||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
|
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
|
||||||
if [[ -f "$pid_file" ]]; then
|
bash "${SCRIPT_DIR}/stop_sglang_docker.sh" "$tp" "$dp" >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
local pid
|
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
# In container-reuse mode, pid is the server PID inside the container.
|
|
||||||
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang server inside container (tp=${tp}, dp=${dp}, pid=${pid})"
|
|
||||||
docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping sglang server pid=${pid} (tp=${tp}, dp=${dp})"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Fallback: remove any Docker container started by this experiment (legacy mode).
|
|
||||||
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
|
||||||
docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Fallback: kill any SGLang launch processes for this model.
|
# Fallback: kill any SGLang launch processes for this model.
|
||||||
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
|
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
|
||||||
@ -89,23 +66,11 @@ build_server_args() {
|
|||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
|
|
||||||
local args=(
|
deploy_render_args \
|
||||||
"python3 -m sglang.launch_server" --model-path "$MODEL_PATH"
|
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
|
||||||
--trust-remote-code
|
"$tp" "$dp" \
|
||||||
--tp-size "$tp"
|
"${SGLANG_PORT:-30031}" \
|
||||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
"$MODEL_PATH"
|
||||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
|
||||||
--context-length "$CONTEXT_LENGTH"
|
|
||||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$SGLANG_PORT"
|
|
||||||
)
|
|
||||||
if [[ "$dp" -gt 1 ]]; then
|
|
||||||
args+=(
|
|
||||||
--dp-size "$dp"
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
printf '%s ' "${args[@]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
start_server() {
|
start_server() {
|
||||||
|
|||||||
@ -1,105 +1,33 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Start SGLang server in Docker for a given TPxDP configuration.
|
# Start the SGLang TPxDP server through the shared deployment layer.
|
||||||
# Usage: start_sglang_docker.sh <TP> <DP>
|
# Usage: start_sglang_docker.sh <TP> <DP>
|
||||||
#
|
set -Eeuo pipefail
|
||||||
# Uses the verified SGLang SM120 image and keeps its JIT cache on
|
|
||||||
# persistent storage. The container is removed automatically on stop.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
TP="${1}"
|
TP="${1:-}"
|
||||||
DP="${2}"
|
DP="${2:-}"
|
||||||
|
if [[ -z "$TP" || -z "$DP" ]]; then
|
||||||
|
echo "Usage: $0 <TP> <DP>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
log "starting SGLang server tp=${TP} dp=${DP} via deploy profile"
|
||||||
PORT="${SGLANG_PORT:-30031}"
|
deploy_start \
|
||||||
NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
|
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
|
||||||
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
|
"$TP" "$DP" \
|
||||||
|
"${RUNTIME_BASE}/logs" \
|
||||||
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
"${SGLANG_PORT:-30031}" \
|
||||||
rm -f "$PID_FILE"
|
"$MODEL_PATH" \
|
||||||
|
"${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
|
||||||
# Clean up any stale container with the same name.
|
|
||||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
|
||||||
|
|
||||||
SERVER_ARGS=(
|
|
||||||
-m sglang.launch_server
|
|
||||||
--model-path "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--tp-size "$TP"
|
|
||||||
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
|
||||||
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
|
||||||
--context-length "$CONTEXT_LENGTH"
|
|
||||||
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$PORT"
|
|
||||||
)
|
|
||||||
|
|
||||||
if [[ "$DP" -gt 1 ]]; then
|
|
||||||
SERVER_ARGS+=(
|
|
||||||
--dp-size "$DP"
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
|
||||||
|
|
||||||
echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="
|
|
||||||
echo "Image: $IMAGE"
|
|
||||||
echo "Model: $MODEL_PATH"
|
|
||||||
echo "Container name: $NAME"
|
|
||||||
echo "Host port: $PORT"
|
|
||||||
echo "Command: python3 ${SERVER_ARGS_STR}"
|
|
||||||
echo "Log: $LOG"
|
|
||||||
|
|
||||||
# Run docker in the foreground so that killing the host process stops the
|
|
||||||
# container (the --rm flag ensures cleanup). nohup lets us background it and
|
|
||||||
# capture the host PID in the same way as the bare-metal start script.
|
|
||||||
nohup docker run --rm \
|
|
||||||
--name "$NAME" \
|
|
||||||
--gpus all \
|
|
||||||
--privileged \
|
|
||||||
--ipc=host \
|
|
||||||
--network host \
|
|
||||||
--ulimit memlock=-1 \
|
|
||||||
--ulimit stack=67108864 \
|
|
||||||
--entrypoint python3 \
|
|
||||||
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
|
||||||
-v "${CACHE_DIR}:/root/.cache" \
|
|
||||||
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
|
||||||
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
|
||||||
-e PYTHONUNBUFFERED=1 \
|
|
||||||
-e HF_HUB_OFFLINE=1 \
|
|
||||||
-e TRANSFORMERS_OFFLINE=1 \
|
|
||||||
-e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \
|
|
||||||
"$IMAGE" \
|
|
||||||
"${SERVER_ARGS[@]}" \
|
|
||||||
> "$LOG" 2>&1 &
|
|
||||||
|
|
||||||
PID=$!
|
|
||||||
echo $PID > "$PID_FILE"
|
|
||||||
echo "PID: $PID"
|
|
||||||
echo "Waiting for health on port ${PORT}..."
|
|
||||||
|
|
||||||
for i in $(seq 1 240); do
|
|
||||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
|
||||||
echo "SGLang server is ready at http://127.0.0.1:${PORT}"
|
|
||||||
echo "Log: $LOG"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
if ! kill -0 $PID 2>/dev/null; then
|
|
||||||
echo "ERROR: Docker SGLang server exited early"
|
|
||||||
tail -200 "$LOG"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "Waiting... ($i/240)"
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ERROR: Docker SGLang server not healthy after 240 retries"
|
|
||||||
tail -200 "$LOG"
|
|
||||||
exit 1
|
|
||||||
|
|||||||
@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stop the SGLang TPxDP server through the shared deployment layer.
|
||||||
|
# Usage: stop_sglang_docker.sh <TP> <DP>
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
TP="${1:-}"
|
||||||
|
DP="${2:-}"
|
||||||
|
if [[ -z "$TP" || -z "$DP" ]]; then
|
||||||
|
echo "Usage: $0 <TP> <DP>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
log "stopping SGLang server tp=${TP} dp=${DP} via deploy profile"
|
||||||
|
deploy_stop \
|
||||||
|
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_sglang_tp_dp_matrix}" \
|
||||||
|
"$TP" "$DP" \
|
||||||
|
"${SGLANG_PORT:-30031}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
|
||||||
@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
|
|||||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}"
|
||||||
|
|
||||||
ENGINE="vllm"
|
ENGINE="vllm"
|
||||||
ENGINE_PORT="$VLLM_PORT"
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
@ -38,20 +42,9 @@ engine_is_healthy() {
|
|||||||
engine_stop_server() {
|
engine_stop_server() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
|
||||||
|
|
||||||
if [[ -f "$pid_file" ]]; then
|
log "stopping vllm server tp=${tp} dp=${dp} via deploy profile"
|
||||||
local pid
|
bash "${SCRIPT_DIR}/stop_vllm_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
fi
|
|
||||||
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
|
||||||
ACTIVE_ENGINE_SERVER_LOG=""
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
sleep 2
|
sleep 2
|
||||||
}
|
}
|
||||||
@ -59,22 +52,7 @@ engine_stop_server() {
|
|||||||
engine_build_server_args() {
|
engine_build_server_args() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local -a args=(
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
vllm serve "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
|
||||||
--block-size "$BLOCK_SIZE"
|
|
||||||
--tensor-parallel-size "$tp"
|
|
||||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
|
||||||
--max-model-len "$MAX_MODEL_LEN"
|
|
||||||
--max-num-seqs "$MAX_NUM_SEQS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$ENGINE_PORT"
|
|
||||||
)
|
|
||||||
if (( dp > 1 )); then
|
|
||||||
args+=(--data-parallel-size "$dp")
|
|
||||||
fi
|
|
||||||
printf '%q ' "${args[@]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
engine_start_server() {
|
engine_start_server() {
|
||||||
|
|||||||
@ -15,6 +15,10 @@ source "${SCRIPT_DIR}/config.env"
|
|||||||
source "${SCRIPT_DIR}/adaptive_config.env"
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}"
|
||||||
|
|
||||||
ENGINE="vllm"
|
ENGINE="vllm"
|
||||||
ENGINE_PORT="$VLLM_PORT"
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
@ -38,20 +42,9 @@ engine_is_healthy() {
|
|||||||
engine_stop_server() {
|
engine_stop_server() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
|
||||||
|
|
||||||
if [[ -f "$pid_file" ]]; then
|
log "stopping vllm server tp=${tp} dp=${dp} via deploy profile"
|
||||||
local pid
|
bash "${SCRIPT_DIR}/stop_vllm_docker.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 5
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
fi
|
|
||||||
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
|
||||||
ACTIVE_ENGINE_SERVER_LOG=""
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
sleep 2
|
sleep 2
|
||||||
}
|
}
|
||||||
@ -59,22 +52,7 @@ engine_stop_server() {
|
|||||||
engine_build_server_args() {
|
engine_build_server_args() {
|
||||||
local tp="$1"
|
local tp="$1"
|
||||||
local dp="$2"
|
local dp="$2"
|
||||||
local -a args=(
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
vllm serve "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
|
||||||
--block-size "$BLOCK_SIZE"
|
|
||||||
--tensor-parallel-size "$tp"
|
|
||||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
|
||||||
--max-model-len "$MAX_MODEL_LEN"
|
|
||||||
--max-num-seqs "$MAX_NUM_SEQS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$ENGINE_PORT"
|
|
||||||
)
|
|
||||||
if (( dp > 1 )); then
|
|
||||||
args+=(--data-parallel-size "$dp")
|
|
||||||
fi
|
|
||||||
printf '%q ' "${args[@]}"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
engine_start_server() {
|
engine_start_server() {
|
||||||
@ -163,6 +141,8 @@ export -f engine_run_bench
|
|||||||
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
|
|
||||||
export SEARCH_START_CONCURRENCY=16
|
export SEARCH_START_CONCURRENCY=16
|
||||||
export SEARCH_ADDEND=16
|
export SEARCH_ADDEND=16
|
||||||
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
@ -171,5 +151,3 @@ export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
|||||||
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
# shapes in this TP/DP group. Zero disables this rule.
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
|
|
||||||
adaptive_main "$@"
|
|
||||||
|
|||||||
@ -1,114 +1,33 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Start vLLM server in Docker for a given TPxDP configuration.
|
# Start the vLLM TPxDP server through the shared deployment layer.
|
||||||
# Usage: start_vllm_docker.sh <TP> <DP>
|
# Usage: start_vllm_docker.sh <TP> <DP>
|
||||||
#
|
set -Eeuo pipefail
|
||||||
# Uses the verified vLLM SM120 image and keeps its cache on persistent
|
|
||||||
# storage. The container is removed automatically on stop.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
TP="${1}"
|
TP="${1:-}"
|
||||||
DP="${2}"
|
DP="${2:-}"
|
||||||
|
if [[ -z "$TP" || -z "$DP" ]]; then
|
||||||
|
echo "Usage: $0 <TP> <DP>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
|
log "starting vLLM server tp=${TP} dp=${DP} via deploy profile"
|
||||||
PORT="${VLLM_PORT:-30030}"
|
deploy_start \
|
||||||
NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
|
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}" \
|
||||||
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
"$TP" "$DP" \
|
||||||
|
"${RUNTIME_BASE}/logs" \
|
||||||
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
"${VLLM_PORT:-30030}" \
|
||||||
rm -f "$PID_FILE"
|
"$MODEL_PATH" \
|
||||||
|
"${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
|
||||||
# Clean up any stale container with the same name.
|
|
||||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
|
||||||
|
|
||||||
SERVER_ARGS=(
|
|
||||||
serve "$MODEL_PATH"
|
|
||||||
--trust-remote-code
|
|
||||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
|
||||||
--block-size "$BLOCK_SIZE"
|
|
||||||
--tensor-parallel-size "$TP"
|
|
||||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
|
||||||
--max-model-len "$MAX_MODEL_LEN"
|
|
||||||
--max-num-seqs "$MAX_NUM_SEQS"
|
|
||||||
--host 0.0.0.0
|
|
||||||
--port "$PORT"
|
|
||||||
)
|
|
||||||
|
|
||||||
if [[ "$DP" -gt 1 ]]; then
|
|
||||||
SERVER_ARGS+=(
|
|
||||||
--data-parallel-size "$DP"
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
|
||||||
|
|
||||||
echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
|
|
||||||
echo "Image: $IMAGE"
|
|
||||||
echo "Model: $MODEL_PATH"
|
|
||||||
echo "Container name: $NAME"
|
|
||||||
echo "Host port: $PORT"
|
|
||||||
echo "Command: vllm ${SERVER_ARGS_STR}"
|
|
||||||
echo "Log: $LOG"
|
|
||||||
|
|
||||||
# Run docker in the foreground so that killing the host process stops the
|
|
||||||
# container (the --rm flag ensures cleanup). nohup lets us background it and
|
|
||||||
# capture the host PID in the same way as the bare-metal start script.
|
|
||||||
nohup docker run --rm \
|
|
||||||
--name "$NAME" \
|
|
||||||
--gpus all \
|
|
||||||
--privileged \
|
|
||||||
--ipc=host \
|
|
||||||
--network host \
|
|
||||||
--ulimit memlock=-1 \
|
|
||||||
--ulimit stack=67108864 \
|
|
||||||
--entrypoint vllm \
|
|
||||||
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
|
||||||
-v "${CACHE_DIR}:/root/.cache" \
|
|
||||||
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
|
||||||
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
|
||||||
-e PYTHONUNBUFFERED=1 \
|
|
||||||
-e HF_HUB_OFFLINE=1 \
|
|
||||||
-e TRANSFORMERS_OFFLINE=1 \
|
|
||||||
"$IMAGE" \
|
|
||||||
"${SERVER_ARGS[@]}" \
|
|
||||||
> "$LOG" 2>&1 &
|
|
||||||
|
|
||||||
PID=$!
|
|
||||||
echo $PID > "$PID_FILE"
|
|
||||||
echo "PID: $PID"
|
|
||||||
echo "Waiting for health on port ${PORT}..."
|
|
||||||
|
|
||||||
for i in $(seq 1 600); do
|
|
||||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
|
||||||
echo "vLLM server is ready at http://127.0.0.1:${PORT}"
|
|
||||||
echo "Log: $LOG"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
# Check if the Docker container is still running (not the nohup PID).
|
|
||||||
# Allow a brief grace period for the container to appear in docker ps.
|
|
||||||
container_running=0
|
|
||||||
for _ in $(seq 1 3); do
|
|
||||||
if docker ps --filter "name=${NAME}" --format '{{.Names}}' | grep -q "^${NAME}$"; then
|
|
||||||
container_running=1
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
if [[ "$container_running" -eq 0 ]]; then
|
|
||||||
echo "ERROR: Docker vLLM container exited early"
|
|
||||||
tail -200 "$LOG"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "Waiting... ($i/600)"
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ERROR: Docker vLLM server not healthy after 600 retries"
|
|
||||||
tail -200 "$LOG"
|
|
||||||
exit 1
|
|
||||||
|
|||||||
@ -0,0 +1,29 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stop the vLLM TPxDP server through the shared deployment layer.
|
||||||
|
# Usage: stop_vllm_docker.sh <TP> <DP>
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
TP="${1:-}"
|
||||||
|
DP="${2:-}"
|
||||||
|
if [[ -z "$TP" || -z "$DP" ]]; then
|
||||||
|
echo "Usage: $0 <TP> <DP>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
log "stopping vLLM server tp=${TP} dp=${DP} via deploy profile"
|
||||||
|
deploy_stop \
|
||||||
|
"${DEPLOY_PROFILE:-pro6000/dsv4_pro6000_vllm_tp_dp_matrix}" \
|
||||||
|
"$TP" "$DP" \
|
||||||
|
"${VLLM_PORT:-30030}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
|
||||||
@ -0,0 +1,130 @@
|
|||||||
|
# GLM-5.2-FP8 多机部署（SGLang TP=16，2×RTX 6000D 节点）
|
||||||
|
|
||||||
|
在 pro6000D.1 + pro6000D.3 两台机器（各 8×RTX 6000D）上多机部署 GLM-5.2-FP8，
|
||||||
|
并对齐 H20 / DSV4 实验的 benchmark 方法做 TP×DP 矩阵测试。
|
||||||
|
|
||||||
|
## 为什么必须多机
|
||||||
|
|
||||||
|
GLM-5.2 是 MoE 模型：**总参数约 735B，FP8 权重约 700GB**（256 路由专家 + 1 共享
|
||||||
|
专家，每 token 激活 8 个，DeepSeek-V3 架构）。单机 8 卡 RTX 6000D 显存共 685GB，
|
||||||
|
**装不下 700GB 权重**，因此 2 节点 16 卡是必需项，不是优化项。
|
||||||
|
|
||||||
|
## 架构
|
||||||
|
|
||||||
|
```
|
||||||
|
HTTP 请求 (bench_serving)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌──────────────┐ NCCL/RoCE (41.6GB/s) ┌──────────────┐
|
||||||
|
│ pro6000D.1 │ ◄══════════════════════► │ pro6000D.3 │
|
||||||
|
│ node-rank 0 │ 计算网 10.101/10.102 │ node-rank 1 │
|
||||||
|
│ 持有 1/2 权重 │ │ 持有 1/2 权重 │
|
||||||
|
│ HTTP :30031 │ │ 不暴露 HTTP │
|
||||||
|
└──────────────┘ └──────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
- **node0（pro6000D.1）**：主节点，对外暴露 HTTP API，接收所有请求。
|
||||||
|
- **node1（pro6000D.3）**：从节点，只做计算，不对外服务。
|
||||||
|
- 两机通过 NCCL 连接：`--dist-init-addr <node0_ip:port> --nnodes 2 --node-rank <0|1>`。
|
||||||
|
- NCCL bootstrap 走管理网（174.1.51.x），数据面走 RoCE 计算网（mlx5_0/mlx5_3）。
|
||||||
|
- benchmark 客户端只打 node0 的 HTTP，感知不到背后是两台机器。
|
||||||
|
|
||||||
|
## 前置条件（两台机器都要满足）
|
||||||
|
|
||||||
|
1. **模型文件**：两机都要有完整的 GLM-5.2-FP8，路径统一为
|
||||||
|
`/data/hf_models/GLM-5.2-FP8`（141 个 safetensors + index.json）。
|
||||||
|
- pro6000D.1：下载中（截至编写时 112/141 分片）。
|
||||||
|
- pro6000D.3：**当前没有，需补齐**（从 .1 rsync 传过去或重新下载）。
|
||||||
|
2. **Docker 镜像**：两机都要有
|
||||||
|
`lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45`（sglang dev，原生支持
|
||||||
|
GLM-5.2 的 `glm_moe_dsa` 架构）。
|
||||||
|
- pro6000D.1：已有。
|
||||||
|
- pro6000D.3：下载中。
|
||||||
|
3. **SSH 互通**：node0 能 `ssh pro6000D.3` 免密登录（已配置）。
|
||||||
|
4. **NCCL 跨机已验证**：双 NIC 调优后 all_reduce busbw 41.6 GB/s（见带宽报告）。
|
||||||
|
|
||||||
|
## 文件说明
|
||||||
|
|
||||||
|
| 文件 | 作用 |
|
||||||
|
|------|------|
|
||||||
|
| `config.env` | 全局配置：模型路径、TP=16、上下文 256K、NCCL 调优、节点拓扑 |
|
||||||
|
| `start_sglang_node.sh <rank>` | 单节点启动（参数化 rank），node0/node1 共用 |
|
||||||
|
| `start_sglang_multinode.sh <tp> <dp>` | 多机编排：sync→起 node1→起 node0→等 health |
|
||||||
|
| `stop_sglang_multinode.sh <tp> <dp>` | 多机停止：停 node0（本地）+ node1（ssh） |
|
||||||
|
| `start_sglang_dp.sh <tp> <dp>` | run_bench 入口，委托给 multinode 编排 |
|
||||||
|
| `run_bench.sh` | TP×DP 矩阵 benchmark（复用 DSV4 模板，改 stop 为多机） |
|
||||||
|
| `matrix.json` | ISL×OSL 测试矩阵（复用 DSV4） |
|
||||||
|
| `adaptive_config.env` | 自适应并发搜索配置（复用 DSV4） |
|
||||||
|
|
||||||
|
## NCCL 调优（关键，不可省）
|
||||||
|
|
||||||
|
跨机必须带以下环境变量（已固化在 config.env，注入容器），否则带宽从 41.6
|
||||||
|
跌到 21 GB/s：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NCCL_IB_HCA=mlx5_0,mlx5_3 # 启用两张 RoCE 网卡
|
||||||
|
NCCL_MIN_NCHANNELS=8 # 强制 8 channel 分流到两卡
|
||||||
|
NCCL_IB_QPS_PER_CONNECTION=4 # 每连接 4 QP 提升 RoCE 并行
|
||||||
|
NCCL_NET_GDR_LEVEL=PHB # GPUDirect RDMA
|
||||||
|
NCCL_SOCKET_IFNAME=eth1 # bootstrap 走管理网网卡
|
||||||
|
```
|
||||||
|
|
||||||
|
## 用法
|
||||||
|
|
||||||
|
### 冒烟测试（单场景验证端到端）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /data/yy/sskj/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16
|
||||||
|
GRID_LIMIT=1 DRY_RUN=0 bash run_bench.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### 全矩阵（low/high 两个并发点）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
CONCURRENCY_SAMPLES=2 bash run_bench.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
### 手动起停（不走 bench）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash start_sglang_dp.sh 16 1 # 起两机，等 node0 health
|
||||||
|
bash stop_sglang_multinode.sh 16 1 # 停两机
|
||||||
|
```
|
||||||
|
|
||||||
|
### 手动单独起 node1（调试）
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh pro6000D.3 'cd /data/yy/sskj/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16 && bash start_sglang_node.sh 1'
|
||||||
|
```
|
||||||
|
|
||||||
|
## 启动流程说明
|
||||||
|
|
||||||
|
`start_sglang_multinode.sh` 做的事：
|
||||||
|
|
||||||
|
1. **rsync 实验目录到 node1**（让 node1 有 config.env + start_sglang_node.sh）
|
||||||
|
2. **ssh 后台起 node1**：`ssh pro6000D.3 '... start_sglang_node.sh 1'`，node1
|
||||||
|
启动容器后进入 rendezvous 等待窗口，等 node0 连接
|
||||||
|
3. **本地起 node0**：`start_sglang_node.sh 0`，node0 发起 NCCL 连接，两机会合后
|
||||||
|
一起加载模型
|
||||||
|
4. **轮询 node0 /health**：就绪后返回（最多等 480×5s=40 分钟，多机加载慢）
|
||||||
|
|
||||||
|
node1 的日志在远端 `/tmp/glm52_node1_inner.log` 和本实验 `runtime/logs/` 下。
|
||||||
|
|
||||||
|
## 与 H20 基线对比
|
||||||
|
|
||||||
|
- H20 基线：`experiments/h20/glm_h20_vllm_tp_dp_matrix`，单机 8 卡 vLLM TP=8。
|
||||||
|
- 本实验：2 机 16 卡 SGLang TP=16。
|
||||||
|
- **框架不同**（vLLM vs SGLang）、**并行不同**（单机 TP=8 vs 多机 TP=16），
|
||||||
|
对比时需注明。但 benchmark 方法一致（sglang bench_serving + 同一 matrix.json
|
||||||
|
+ 自适应并发），吞吐/延迟指标可对照。
|
||||||
|
- 用 `compare.py` 对比两边 results。
|
||||||
|
|
||||||
|
## 注意事项
|
||||||
|
|
||||||
|
- **单机 TP=8 装不下**：735GB 权重 ÷ 8 卡 ≈ 92GB/卡 > 85.6GB 单卡上限，会 OOM。
|
||||||
|
必须多机。
|
||||||
|
- **CUDA 版本不一致**（.1 是 12.8、.3 是 13.0）不影响：两机都在 cu130 容器内跑，
|
||||||
|
宿主机 CUDA 版本无关。
|
||||||
|
- **node1 不暴露 HTTP**：所有请求打 node0，node1 纯计算。
|
||||||
|
- **加载慢**：700GB 权重 + NCCL 建连，首次加载预计 10-20 分钟，health 轮询
|
||||||
|
上限设了 40 分钟。
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
# Adaptive concurrency search settings.
|
||||||
|
#
|
||||||
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
||||||
|
# C = start, start * multiplier, ... up to max
|
||||||
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
||||||
|
# PLATEAU_PATIENCE consecutive points.
|
||||||
|
|
||||||
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||||
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}"
|
||||||
|
|
||||||
|
# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
|
||||||
|
ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
|
||||||
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||||
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||||
|
|
||||||
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
||||||
|
# low-gain points prevent one noisy measurement from stopping the search.
|
||||||
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||||
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||||
|
|
||||||
|
# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
|
||||||
|
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
|
||||||
|
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||||
|
|
||||||
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
||||||
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
||||||
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
||||||
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
||||||
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
||||||
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||||
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
||||||
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
||||||
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
||||||
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
||||||
|
|
||||||
|
# Reject a point if the completed request count or actual token lengths do not
|
||||||
|
# match the requested workload.
|
||||||
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
||||||
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
||||||
|
|
||||||
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
||||||
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Optional space-separated filters, useful for smoke tests:
|
||||||
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
||||||
|
TP_LIST="${TP_LIST:-}"
|
||||||
|
ISL_LIST="${ISL_LIST:-}"
|
||||||
|
OSL_LIST="${OSL_LIST:-}"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
162
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/compare.py
Executable file
162
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/compare.py
Executable file
@ -0,0 +1,162 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Cross TP×DP configuration comparison for dsv4_h200_sglang_tp_dp_matrix.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 compare.py --run-root results/<run_id> [--output comparison.md]
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def load_result(result_root: Path) -> dict:
|
||||||
|
path = result_root / "results.json"
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
|
||||||
|
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
|
||||||
|
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
||||||
|
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
||||||
|
if ttft_ok and tpot_ok:
|
||||||
|
return "PASS"
|
||||||
|
if ttft_ok or tpot_ok:
|
||||||
|
return "PARTIAL"
|
||||||
|
return "FAIL"
|
||||||
|
|
||||||
|
|
||||||
|
def gpu_memory_str(gpu: dict | None) -> str:
|
||||||
|
if not gpu:
|
||||||
|
return "-"
|
||||||
|
peak = gpu.get("peak_used_mb", 0)
|
||||||
|
total = gpu.get("memory_total_mb", 0)
|
||||||
|
if total:
|
||||||
|
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
|
||||||
|
return f"{peak:.0f}"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--run-root", type=Path, required=True)
|
||||||
|
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||||
|
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
||||||
|
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# Discover configurations: tp*_dp* directories.
|
||||||
|
configs = []
|
||||||
|
for subdir in sorted(args.run_root.iterdir()):
|
||||||
|
if not subdir.is_dir():
|
||||||
|
continue
|
||||||
|
name = subdir.name
|
||||||
|
if not (name.startswith("tp") and "_dp" in name):
|
||||||
|
continue
|
||||||
|
results_json = subdir / "results.json"
|
||||||
|
if not results_json.exists():
|
||||||
|
continue
|
||||||
|
configs.append((name, load_result(subdir)))
|
||||||
|
|
||||||
|
if not configs:
|
||||||
|
print(f"No tp*_dp* results found under {args.run_root}")
|
||||||
|
return
|
||||||
|
|
||||||
|
model = configs[0][1].get("metadata", {}).get("model", "unknown")
|
||||||
|
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
|
||||||
|
|
||||||
|
# Group by scenario name.
|
||||||
|
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
|
||||||
|
skipped: dict[str, dict[str, str]] = defaultdict(dict)
|
||||||
|
for label, data in configs:
|
||||||
|
for s in data.get("scenarios", []):
|
||||||
|
key = s["name"]
|
||||||
|
if s.get("status") == "skipped_oom":
|
||||||
|
skipped[key][label] = s.get("note", "skipped")
|
||||||
|
else:
|
||||||
|
by_scenario[key][label] = s
|
||||||
|
|
||||||
|
with open(args.output, "w", encoding="utf-8") as f:
|
||||||
|
f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n")
|
||||||
|
f.write("## Summary\n\n")
|
||||||
|
f.write(f"- Model: `{model}`\n")
|
||||||
|
f.write(f"- Hardware: {hardware}\n")
|
||||||
|
f.write("- Backend: SGLang (Docker)\n")
|
||||||
|
f.write("- Benchmark client: `sglang.benchmark.serving`\n")
|
||||||
|
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
||||||
|
|
||||||
|
# Configuration overview.
|
||||||
|
f.write("### Configurations\n\n")
|
||||||
|
f.write("| Config | TP | DP | GPUs/replica | Notes |\n")
|
||||||
|
f.write("|---|---:|---:|---:|---|\n")
|
||||||
|
for label, data in configs:
|
||||||
|
cfg = data.get("config", {})
|
||||||
|
tp = cfg.get("tp", "?")
|
||||||
|
dp = cfg.get("dp", "?")
|
||||||
|
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
# Side-by-side table.
|
||||||
|
f.write("## Side-by-side results\n\n")
|
||||||
|
headers = [
|
||||||
|
"Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s",
|
||||||
|
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
|
||||||
|
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
|
||||||
|
]
|
||||||
|
f.write("| " + " | ".join(headers) + " |\n")
|
||||||
|
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
|
||||||
|
|
||||||
|
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))):
|
||||||
|
_, isl, dsl = re.findall(r"\d+", scenario_name)
|
||||||
|
for label, data in configs:
|
||||||
|
s = by_scenario[scenario_name].get(label)
|
||||||
|
if s is None:
|
||||||
|
if scenario_name in skipped and label in skipped[scenario_name]:
|
||||||
|
note = skipped[scenario_name][label]
|
||||||
|
f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
|
||||||
|
continue
|
||||||
|
cfg = s["config"]
|
||||||
|
m = s["metrics"]
|
||||||
|
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||||
|
gpu = m.get("gpu_memory")
|
||||||
|
f.write(
|
||||||
|
f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | "
|
||||||
|
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||||
|
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||||
|
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||||
|
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Best throughput per ISL/DSL.
|
||||||
|
f.write("\n## Best throughput per (ISL, DSL)\n\n")
|
||||||
|
f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
|
||||||
|
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
|
||||||
|
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
|
||||||
|
for scenario_name, backends in by_scenario.items():
|
||||||
|
_, isl, dsl = re.findall(r"\d+", scenario_name)
|
||||||
|
isl_i, dsl_i = int(isl), int(dsl)
|
||||||
|
for label, s in backends.items():
|
||||||
|
m = s["metrics"]
|
||||||
|
out_tok = m["output_token_throughput"]
|
||||||
|
if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]:
|
||||||
|
best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s)
|
||||||
|
for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
|
||||||
|
m = s["metrics"]
|
||||||
|
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||||
|
f.write(
|
||||||
|
f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | "
|
||||||
|
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
f.write("\n## Notes\n\n")
|
||||||
|
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
||||||
|
f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
|
||||||
|
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
|
||||||
|
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
|
||||||
|
|
||||||
|
print(f"Wrote comparison to {args.output}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
145
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/config.env
Executable file
145
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/config.env
Executable file
@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Multi-node SGLang deployment for GLM-5.2-FP8 on 2x RTX 6000D nodes (16 GPUs).
|
||||||
|
#
|
||||||
|
# Topology:
|
||||||
|
# node0 = pro6000D.1 (gpu1, 174.1.51.5, 10.101.0.11 / 10.102.0.11)
|
||||||
|
# node1 = pro6000D.3 (gpu3, 174.1.51.7, 10.101.0.13 / 10.102.0.13)
|
||||||
|
# Each node has 8x RTX 6000D (85.6GB) = 685GB; GLM-5.2-FP8 is ~700GB, so a
|
||||||
|
# single node cannot hold the weights -> 2-node TP=16 is mandatory.
|
||||||
|
#
|
||||||
|
# SGLang multi-node: every node runs its own `python3 -m sglang.launch_server`
|
||||||
|
# and they connect via NCCL using --dist-init-addr/--nnodes/--node-rank.
|
||||||
|
# node0 exposes the HTTP API; node1 is compute-only. The benchmark client only
|
||||||
|
# talks to node0.
|
||||||
|
|
||||||
|
# --- Path bootstrap -------------------------------------------------------
|
||||||
|
# run_bench.sh sources platform.sh (which sets ROOT_DIR) before this file, but
|
||||||
|
# the start_* scripts source this file directly. Provide a fallback so
|
||||||
|
# ${ROOT_DIR} / ${SCRIPT_DIR} expand correctly in both paths.
|
||||||
|
SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)}"
|
||||||
|
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../../.." 2>/dev/null && pwd)}"
|
||||||
|
# If the relative climb failed (e.g. dir moved), fall back to the script dir.
|
||||||
|
[[ -z "$ROOT_DIR" || ! -d "$ROOT_DIR/platforms" ]] && ROOT_DIR="$SCRIPT_DIR"
|
||||||
|
|
||||||
|
EXPERIMENT="glm52_pro6000_sglang_multinode_tp16"
|
||||||
|
MODEL_NAME="GLM-5.2-FP8"
|
||||||
|
MODEL_PATH="/data/hf_models/GLM-5.2-FP8"
|
||||||
|
SERVED_MODEL_NAME="GLM-5.2-FP8"
|
||||||
|
|
||||||
|
SGLANG_PORT="${SGLANG_PORT:-30031}"
|
||||||
|
|
||||||
|
# Python interpreter for orchestration scripts (parse_backend.py, compare.py,
|
||||||
|
# etc.) and the benchmark client. Defaults to the system python3 if the sglang
|
||||||
|
# venv does not exist on the host.
|
||||||
|
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
|
||||||
|
|
||||||
|
# Run the benchmark client natively (0) or inside Docker (1).
|
||||||
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
|
# All 8 GPUs per node participate in TP=16 (8 per node x 2 nodes).
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||||
|
|
||||||
|
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||||||
|
# directory under this experiment so the benchmark is self-contained.
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Multi-node topology
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Management network is used for the NCCL bootstrap (dist-init-addr). The data
|
||||||
|
# plane (NCCL/RoCE) is selected automatically by NCCL inside the container via
|
||||||
|
# the NCCL_* env vars below.
|
||||||
|
NODE0_HOST="${NODE0_HOST:-pro6000D.1}"
|
||||||
|
NODE1_HOST="${NODE1_HOST:-pro6000D.3}"
|
||||||
|
NODE0_IP="${NODE0_IP:-174.1.51.5}"
|
||||||
|
NODE1_IP="${NODE1_IP:-174.1.51.7}"
|
||||||
|
NNODES="${NNODES:-2}"
|
||||||
|
# Port for NCCL bootstrap (dist-init-addr). Must be free on node0.
|
||||||
|
DIST_INIT_PORT="${DIST_INIT_PORT:-50000}"
|
||||||
|
# SSH alias for reaching node1 from the control machine (node0).
|
||||||
|
NODE1_SSH="${NODE1_SSH:-root@174.1.51.7}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Parallel configuration. TP=16 across 2 nodes (8 GPUs/node).
|
||||||
|
# Constraint (sglang): tp_size * pp_size must be divisible by nnodes.
|
||||||
|
# 16 * 1 = 16, 16 % 2 == 0 OK
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
TP_SIZE="${TP_SIZE:-16}"
|
||||||
|
DP_SIZE="${DP_SIZE:-1}"
|
||||||
|
# PARALLEL_CONFIGS is kept for compatibility with the matrix runner; the single
|
||||||
|
# multi-node config is "16 1".
|
||||||
|
declare -a PARALLEL_CONFIGS=(
|
||||||
|
"16 1"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# SGLang server settings (aligned with dsv4_pro6000_sglang_tp_dp_matrix).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"
|
||||||
|
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
|
||||||
|
# TileLang DSA on SM120 currently requires BF16 KV cache. Keep the smoke-test
|
||||||
|
# context at 128K so the doubled KV-cache element size fits beside FP8 weights.
|
||||||
|
CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
|
||||||
|
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
|
||||||
|
# GLM-5.2 is shipped as FP8 weights; tell SGLang explicitly.
|
||||||
|
QUANTIZATION="${QUANTIZATION:-fp8}"
|
||||||
|
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-bfloat16}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# NCCL tuning for cross-node RoCE (mandatory, otherwise bandwidth collapses).
|
||||||
|
# Verified in the bandwidth report: dual-NIC tuned all_reduce = 41.6 GB/s,
|
||||||
|
# vs 21 GB/s without these knobs.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
export NCCL_IB_HCA="${NCCL_IB_HCA:-mlx5_0,mlx5_3}"
|
||||||
|
export NCCL_MIN_NCHANNELS="${NCCL_MIN_NCHANNELS:-8}"
|
||||||
|
export NCCL_IB_QPS_PER_CONNECTION="${NCCL_IB_QPS_PER_CONNECTION:-4}"
|
||||||
|
export NCCL_NET_GDR_LEVEL="${NCCL_NET_GDR_LEVEL:-PHB}"
|
||||||
|
# Bind the NCCL socket (bootstrap/OOB) to the management NIC eth1 (174.1.51.x).
|
||||||
|
# eth0/eth3 are the RoCE data-plane NICs; eth1 carries the mgmt IP.
|
||||||
|
export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth1}"
|
||||||
|
export NCCL_IB_RETRY_CNT="${NCCL_IB_RETRY_CNT:-14}"
|
||||||
|
export NCCL_IB_TIMEOUT="${NCCL_IB_TIMEOUT:-16}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Deployment switch. 0 = native sglang venv, 1 = Docker.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
|
||||||
|
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||||
|
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||||||
|
CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_nightly_cu13_cache}"
|
||||||
|
|
||||||
|
# Matrix and concurrency rules are defined in matrix.json by default.
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
|
||||||
|
# Sampling density for concurrency.
|
||||||
|
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
||||||
|
# 2 = only test the low and high endpoints.
|
||||||
|
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||||||
|
|
||||||
|
# Per-scenario timeout to avoid hangs (seconds). Multi-node load is slower, so
|
||||||
|
# the default is raised.
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-2400}"
|
||||||
|
|
||||||
|
# GPU memory sampling interval (seconds).
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Dry-run mode: if 1, only log the server args and scenario plan without
|
||||||
|
# starting any server or sending requests.
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
|
||||||
|
# Per-config scenario limit for quick smoke tests. 0 = run all generated
|
||||||
|
# scenarios.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
|
|
||||||
|
# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
|
||||||
|
# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
|
||||||
|
# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
|
||||||
|
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True,max_split_size_mb:256}"
|
||||||
|
|
||||||
|
# Seconds to wait for node1 to come up and register before starting node0.
|
||||||
|
NODE1_READINESS_WAIT_S="${NODE1_READINESS_WAIT_S:-15}"
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate the scenario list for the TP×DP matrix experiment.
|
||||||
|
|
||||||
|
Reads matrix.json and prints TSV lines:
|
||||||
|
|
||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
|
||||||
|
mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def sample_concurrency(low: int, high: int, target: int) -> list[int]:
|
||||||
|
"""Return only the low and high concurrency values in [low, high].
|
||||||
|
|
||||||
|
For the TP×DP matrix we only need the two endpoints of the concurrency
|
||||||
|
range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for
|
||||||
|
API compatibility but is ignored.
|
||||||
|
"""
|
||||||
|
assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
|
||||||
|
if low == high:
|
||||||
|
return [low]
|
||||||
|
return [low, high]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
|
||||||
|
with open(matrix_path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
matrix = data["matrix"]
|
||||||
|
concurrency_cfg = data["concurrency"]
|
||||||
|
|
||||||
|
scenarios = []
|
||||||
|
for isl_str in sorted(matrix.keys(), key=int):
|
||||||
|
dsl_map = matrix[isl_str]
|
||||||
|
low = concurrency_cfg[isl_str]["low"]
|
||||||
|
high = concurrency_cfg[isl_str]["high"]
|
||||||
|
concurrencies = sample_concurrency(low, high, target_samples)
|
||||||
|
|
||||||
|
for dsl_str in sorted(dsl_map.keys(), key=int):
|
||||||
|
mark = dsl_map[dsl_str]
|
||||||
|
if mode == "Y" and mark != "Y":
|
||||||
|
continue
|
||||||
|
if mode == "Y+P" and mark not in ("Y", "P"):
|
||||||
|
continue
|
||||||
|
# mode == "all" keeps everything, including N.
|
||||||
|
|
||||||
|
for conc in concurrencies:
|
||||||
|
scenarios.append(
|
||||||
|
{
|
||||||
|
"mark": mark,
|
||||||
|
"input_len": int(isl_str),
|
||||||
|
"output_len": int(dsl_str),
|
||||||
|
"concurrency": conc,
|
||||||
|
"num_prompts": conc * 5,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return scenarios
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--matrix", type=Path, default=Path("matrix.json"))
|
||||||
|
parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None,
|
||||||
|
help="Scenario selection mode. Defaults to matrix.mode.")
|
||||||
|
parser.add_argument("--target-samples", type=int, default=0,
|
||||||
|
help="Target number of concurrency samples. 0 = heuristic (6-8).")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
with open(args.matrix, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
mode = args.mode if args.mode else data.get("mode", "Y+P")
|
||||||
|
|
||||||
|
target_samples = args.target_samples
|
||||||
|
if target_samples <= 0:
|
||||||
|
env_samples = os.getenv("CONCURRENCY_SAMPLES", "0")
|
||||||
|
try:
|
||||||
|
target_samples = int(env_samples)
|
||||||
|
except ValueError:
|
||||||
|
target_samples = 0
|
||||||
|
if target_samples <= 0:
|
||||||
|
target_samples = 7
|
||||||
|
|
||||||
|
scenarios = generate_scenarios(args.matrix, mode, target_samples)
|
||||||
|
|
||||||
|
print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts")
|
||||||
|
for s in scenarios:
|
||||||
|
print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,128 @@
|
|||||||
|
{
|
||||||
|
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
|
||||||
|
"mode": "Y",
|
||||||
|
"matrix": {
|
||||||
|
"1024": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "P",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "P",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "P",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"concurrency": {
|
||||||
|
"1024": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 128
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 64
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 64
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 32
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 16
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 8
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-134240",
|
||||||
|
"timestamp": "2026-07-24T13:42:41+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-143107",
|
||||||
|
"timestamp": "2026-07-24T14:31:22+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-150539",
|
||||||
|
"timestamp": "2026-07-24T15:05:40+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-151233",
|
||||||
|
"timestamp": "2026-07-24T15:12:34+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-152532",
|
||||||
|
"timestamp": "2026-07-24T15:25:33+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-153034",
|
||||||
|
"timestamp": "2026-07-24T15:30:34+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-154223",
|
||||||
|
"timestamp": "2026-07-24T15:42:24+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-154918",
|
||||||
|
"timestamp": "2026-07-24T15:49:19+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": []
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-160131",
|
||||||
|
"timestamp": "2026-07-24T16:01:33+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": []
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-161611",
|
||||||
|
"timestamp": "2026-07-24T16:16:12+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-161700",
|
||||||
|
"timestamp": "2026-07-24T16:17:00+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,20 @@
|
|||||||
|
# SGLang TP×DP matrix comparison (8x NVIDIA RTX 6000D 84GB)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
- Model: `/data/hf_models/GLM-5.2-FP8`
|
||||||
|
- Hardware: 8x NVIDIA RTX 6000D 84GB
|
||||||
|
- Backend: SGLang (Docker)
|
||||||
|
- Benchmark client: `sglang.benchmark.serving`
|
||||||
|
- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms
|
||||||
|
|
||||||
|
### Configurations
|
||||||
|
|
||||||
|
| Config | TP | DP | GPUs/replica | Notes |
|
||||||
|
|---|---:|---:|---:|---|
|
||||||
|
| tp16_dp1 | ? | ? | ? | server args recorded per ISL in results.json |
|
||||||
|
|
||||||
|
## Side-by-side results
|
||||||
|
|
||||||
|
| Scenario | ISL | DSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO |
|
||||||
|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||||
@ -0,0 +1,489 @@
|
|||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"experiment": "glm52_pro6000_sglang_multinode_tp16_tp16_dp1",
|
||||||
|
"run_id": "20260724-164734",
|
||||||
|
"timestamp": "2026-07-24T16:47:35+08:00",
|
||||||
|
"model": "/data/hf_models/GLM-5.2-FP8",
|
||||||
|
"backend": "sglang",
|
||||||
|
"engine": "sglang",
|
||||||
|
"hardware": "8x NVIDIA RTX 6000D 84GB",
|
||||||
|
"accelerator": "NVIDIA RTX 6000D",
|
||||||
|
"chip": "nvidia_rtx6000d",
|
||||||
|
"script": "experiments/glm52_pro6000_sglang_multinode_tp16/run_bench.sh",
|
||||||
|
"env": "lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45",
|
||||||
|
"git_commit": "d13f61f",
|
||||||
|
"git_dirty": "dirty",
|
||||||
|
"description": "H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c128_i1024_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i4096_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c64_i8192_o4096",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c32_i16384_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c16_i32768_o2048",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c8_i65536_o1024",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c4_i131072_o512",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i262144_o256",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i524288_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c1_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "c2_i1048576_o128",
|
||||||
|
"config": "",
|
||||||
|
"status": "SKIPPED_SERVICE_START_FAILED",
|
||||||
|
"note": "service failed to start"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@ -0,0 +1,79 @@
|
|||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
Y 1024 128 1 5
|
||||||
|
Y 1024 128 128 640
|
||||||
|
Y 1024 256 1 5
|
||||||
|
Y 1024 256 128 640
|
||||||
|
Y 1024 512 1 5
|
||||||
|
Y 1024 512 128 640
|
||||||
|
Y 1024 1024 1 5
|
||||||
|
Y 1024 1024 128 640
|
||||||
|
Y 1024 2048 1 5
|
||||||
|
Y 1024 2048 128 640
|
||||||
|
Y 1024 4096 1 5
|
||||||
|
Y 1024 4096 128 640
|
||||||
|
Y 4096 128 1 5
|
||||||
|
Y 4096 128 64 320
|
||||||
|
Y 4096 256 1 5
|
||||||
|
Y 4096 256 64 320
|
||||||
|
Y 4096 512 1 5
|
||||||
|
Y 4096 512 64 320
|
||||||
|
Y 4096 1024 1 5
|
||||||
|
Y 4096 1024 64 320
|
||||||
|
Y 4096 2048 1 5
|
||||||
|
Y 4096 2048 64 320
|
||||||
|
Y 4096 4096 1 5
|
||||||
|
Y 4096 4096 64 320
|
||||||
|
Y 8192 128 1 5
|
||||||
|
Y 8192 128 64 320
|
||||||
|
Y 8192 256 1 5
|
||||||
|
Y 8192 256 64 320
|
||||||
|
Y 8192 512 1 5
|
||||||
|
Y 8192 512 64 320
|
||||||
|
Y 8192 1024 1 5
|
||||||
|
Y 8192 1024 64 320
|
||||||
|
Y 8192 2048 1 5
|
||||||
|
Y 8192 2048 64 320
|
||||||
|
Y 8192 4096 1 5
|
||||||
|
Y 8192 4096 64 320
|
||||||
|
Y 16384 128 1 5
|
||||||
|
Y 16384 128 32 160
|
||||||
|
Y 16384 256 1 5
|
||||||
|
Y 16384 256 32 160
|
||||||
|
Y 16384 512 1 5
|
||||||
|
Y 16384 512 32 160
|
||||||
|
Y 16384 1024 1 5
|
||||||
|
Y 16384 1024 32 160
|
||||||
|
Y 16384 2048 1 5
|
||||||
|
Y 16384 2048 32 160
|
||||||
|
Y 32768 128 1 5
|
||||||
|
Y 32768 128 16 80
|
||||||
|
Y 32768 256 1 5
|
||||||
|
Y 32768 256 16 80
|
||||||
|
Y 32768 512 1 5
|
||||||
|
Y 32768 512 16 80
|
||||||
|
Y 32768 1024 1 5
|
||||||
|
Y 32768 1024 16 80
|
||||||
|
Y 32768 2048 1 5
|
||||||
|
Y 32768 2048 16 80
|
||||||
|
Y 65536 128 1 5
|
||||||
|
Y 65536 128 8 40
|
||||||
|
Y 65536 256 1 5
|
||||||
|
Y 65536 256 8 40
|
||||||
|
Y 65536 512 1 5
|
||||||
|
Y 65536 512 8 40
|
||||||
|
Y 65536 1024 1 5
|
||||||
|
Y 65536 1024 8 40
|
||||||
|
Y 131072 128 1 5
|
||||||
|
Y 131072 128 4 20
|
||||||
|
Y 131072 256 1 5
|
||||||
|
Y 131072 256 4 20
|
||||||
|
Y 131072 512 1 5
|
||||||
|
Y 131072 512 4 20
|
||||||
|
Y 262144 128 1 5
|
||||||
|
Y 262144 128 2 10
|
||||||
|
Y 262144 256 1 5
|
||||||
|
Y 262144 256 2 10
|
||||||
|
Y 524288 128 1 5
|
||||||
|
Y 524288 128 2 10
|
||||||
|
Y 1048576 128 1 5
|
||||||
|
Y 1048576 128 2 10
|
||||||
|
@ -0,0 +1,154 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
|
ENGINE="sglang"
|
||||||
|
ENGINE_PORT="$SGLANG_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
|
||||||
|
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting sglang server tp=${tp} dp=${dp}"
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
else
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
fi
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
fi
|
||||||
|
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend sglang
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
|
||||||
|
: # random-ids does not need --tokenize-prompt
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
"$DOCKER_IMAGE" \
|
||||||
|
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
@ -0,0 +1,163 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
|
ENGINE="sglang"
|
||||||
|
ENGINE_PORT="$SGLANG_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
log "stopping sglang server tp=${tp} dp=${dp} via deploy profile"
|
||||||
|
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" >> "${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
deploy_render_args "$DEPLOY_PROFILE" "$tp" "$dp" "$ENGINE_PORT" "$MODEL_PATH"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting sglang server tp=${tp} dp=${dp}"
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
else
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
fi
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ -z "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
fi
|
||||||
|
log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend sglang
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
|
||||||
|
: # random-ids does not need --tokenize-prompt
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
"$DOCKER_IMAGE" \
|
||||||
|
-m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
|
|
||||||
|
export SEARCH_START_CONCURRENCY=16
|
||||||
|
export SEARCH_ADDEND=16
|
||||||
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
|
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
||||||
|
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
||||||
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
536
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/run_bench.sh
Executable file
536
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/run_bench.sh
Executable file
@ -0,0 +1,536 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# TP×DP matrix benchmark for DeepSeek-V4-Flash on SGLang (Docker).
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||||
|
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
|
|
||||||
|
# Export variables used inside functions that are called via bash -c subshells.
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT SGLANG_BENCH_MODULE
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||||||
|
|
||||||
|
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
||||||
|
mkdir -p "$log_dir_global"
|
||||||
|
log_init "${log_dir_global}/orchestrator.log"
|
||||||
|
|
||||||
|
log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
|
||||||
|
log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
is_server_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 10 "http://127.0.0.1:${SGLANG_PORT}/v1/models" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
log "stopping multi-node sglang (tp=${tp}, dp=${dp}) on both nodes"
|
||||||
|
# Delegate to the multi-node stop orchestrator, which tears down node0
|
||||||
|
# (local) and node1 (remote via SSH) together: pid file, docker rm, pkill.
|
||||||
|
bash "${SCRIPT_DIR}/stop_sglang_multinode.sh" "$tp" "$dp" \
|
||||||
|
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.stop.log" 2>&1 || true
|
||||||
|
}
|
||||||
|
|
||||||
|
build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
local args=(
|
||||||
|
"python3 -m sglang.launch_server" --model-path "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--tp-size "$tp"
|
||||||
|
--dp-size "$dp"
|
||||||
|
--quantization "$QUANTIZATION"
|
||||||
|
--moe-runner-backend "$MOE_RUNNER_BACKEND"
|
||||||
|
--mem-fraction-static "$MEM_FRACTION_STATIC"
|
||||||
|
--context-length "$CONTEXT_LENGTH"
|
||||||
|
--max-running-requests "$MAX_RUNNING_REQUESTS"
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$SGLANG_PORT"
|
||||||
|
--dist-init-addr "${NODE0_IP}:${DIST_INIT_PORT}"
|
||||||
|
--nnodes "$NNODES"
|
||||||
|
--node-rank 0
|
||||||
|
)
|
||||||
|
printf '%s ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
log "starting sglang server tp=${tp} dp=${dp}"
|
||||||
|
if [[ -n "${CONTAINER_NAME:-}" ]]; then
|
||||||
|
bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" \
|
||||||
|
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||||
|
else
|
||||||
|
bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" \
|
||||||
|
>> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! is_server_healthy; then
|
||||||
|
log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}"
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
sleep 10
|
||||||
|
start_server "$tp" "$dp"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_bench_serving() {
|
||||||
|
# Inject the offline workload choice consistently for warmup and main runs.
|
||||||
|
local -a dataset_args=(--dataset-name "$BENCH_DATASET_NAME")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
dataset_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
|
||||||
|
: # random-ids does not need --tokenize-prompt
|
||||||
|
else
|
||||||
|
dataset_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then
|
||||||
|
local vol_args=()
|
||||||
|
vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${vol_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
--entrypoint python3 \
|
||||||
|
"${DOCKER_IMAGE}" \
|
||||||
|
-m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m "$SGLANG_BENCH_MODULE" "${dataset_args[@]}" "$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
export -f run_bench_serving
|
||||||
|
|
||||||
|
run_warmup() {
|
||||||
|
local input_len="$1"
|
||||||
|
local output_len="$2"
|
||||||
|
|
||||||
|
log "warming up (input=${input_len}, output=${output_len}, num=1)"
|
||||||
|
bash -c '
|
||||||
|
run_bench_serving \
|
||||||
|
--backend sglang \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port "'"$SGLANG_PORT"'" \
|
||||||
|
--random-input-len "'"$input_len"'" \
|
||||||
|
--random-output-len "'"$output_len"'" \
|
||||||
|
--num-prompts 1 \
|
||||||
|
--max-concurrency 1 \
|
||||||
|
--request-rate 10000 \
|
||||||
|
--output-file /dev/null \
|
||||||
|
--output-details \
|
||||||
|
>> "'"${log_dir_global}/warmup.log"'" 2>&1
|
||||||
|
'
|
||||||
|
log "warmup completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
scenario_already_completed() {
|
||||||
|
local output_file="$1"
|
||||||
|
local expected="$2"
|
||||||
|
[[ -s "$output_file" ]] || return 1
|
||||||
|
local completed
|
||||||
|
completed="$("$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
path = sys.argv[1]
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if line:
|
||||||
|
data = json.loads(line)
|
||||||
|
print(data.get('completed', 0))
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
print(0)
|
||||||
|
" "$output_file")"
|
||||||
|
[[ "${completed:-0}" -ge "$expected" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
scenario_already_processed() {
|
||||||
|
local result_root="$1"
|
||||||
|
local scenario_name="$2"
|
||||||
|
local json_path="${result_root}/results.json"
|
||||||
|
[[ -f "$json_path" ]] || return 1
|
||||||
|
"$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
path, name = sys.argv[1], sys.argv[2]
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
for s in data.get('scenarios', []):
|
||||||
|
if s.get('name') == name:
|
||||||
|
if s.get('status') or s.get('metrics', {}).get('success', 0) > 0:
|
||||||
|
sys.exit(0)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
sys.exit(1)
|
||||||
|
" "$json_path" "$scenario_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local server_outer_log="$2"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
start_gpu_monitor() {
|
||||||
|
local csv_path="$1"
|
||||||
|
mkdir -p "$(dirname "$csv_path")"
|
||||||
|
nvidia-smi \
|
||||||
|
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \
|
||||||
|
--format=csv \
|
||||||
|
-l "$GPU_MEM_SAMPLE_INTERVAL_S" \
|
||||||
|
> "$csv_path" 2>/dev/null &
|
||||||
|
echo $!
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_gpu_monitor() {
|
||||||
|
local pid="$1"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
append_scenario_record() {
|
||||||
|
local result_root="$1"
|
||||||
|
local json_path="$result_root/results.json"
|
||||||
|
shift
|
||||||
|
local scenario_json
|
||||||
|
scenario_json="$("$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||||
|
d = {}
|
||||||
|
for k, v in pairs:
|
||||||
|
try:
|
||||||
|
d[k] = json.loads(v)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
d[k] = v
|
||||||
|
print(json.dumps(d, ensure_ascii=False))
|
||||||
|
" "$@")"
|
||||||
|
PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
|
||||||
|
}
|
||||||
|
|
||||||
|
record_skipped_csv() {
|
||||||
|
local csv_path="$1"
|
||||||
|
shift
|
||||||
|
# Args: key=value
|
||||||
|
local row
|
||||||
|
row="$("$PYTHON" -c "
|
||||||
|
import csv, json, sys, io
|
||||||
|
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||||
|
d = {}
|
||||||
|
for k, v in pairs:
|
||||||
|
try:
|
||||||
|
d[k] = json.loads(v)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
d[k] = v
|
||||||
|
buf = io.StringIO()
|
||||||
|
writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','dsl','concurrency','status','reason','detail_log'], extrasaction='ignore')
|
||||||
|
writer.writerow(d)
|
||||||
|
print(buf.getvalue().strip())
|
||||||
|
" "$@")"
|
||||||
|
echo "$row" >> "$csv_path"
|
||||||
|
}
|
||||||
|
|
||||||
|
skip_remaining_scenarios() {
|
||||||
|
local result_root="$1"
|
||||||
|
local scenario_tsv="$2"
|
||||||
|
local start_index="$3"
|
||||||
|
local status="$4"
|
||||||
|
local reason="$5"
|
||||||
|
local tp="$6"
|
||||||
|
local dp="$7"
|
||||||
|
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||||
|
|
||||||
|
local i=0
|
||||||
|
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
|
||||||
|
if (( i < start_index )); then
|
||||||
|
i=$((i + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
i=$((i + 1))
|
||||||
|
local sname="c${conc}_i${isl}_o${dsl}"
|
||||||
|
if scenario_already_processed "$result_root" "$sname"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"${status}\"" \
|
||||||
|
"note=\"${reason}\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=${status}" "reason=${reason}"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Per-configuration runner
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
run_parallel_config() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local config_label="tp${tp}_dp${dp}"
|
||||||
|
local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}"
|
||||||
|
local raw_dir="${result_root}/raw_outputs"
|
||||||
|
local gpu_log_dir="${result_root}/gpu_logs"
|
||||||
|
local phase_log_dir="${result_root}/logs"
|
||||||
|
mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir"
|
||||||
|
|
||||||
|
log "===== ${config_label} START ====="
|
||||||
|
|
||||||
|
# Generate scenario list for this config.
|
||||||
|
local scenario_tsv="${result_root}/scenarios.tsv"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \
|
||||||
|
--matrix "$MATRIX_FILE" \
|
||||||
|
--mode "$MATRIX_MODE" \
|
||||||
|
> "$scenario_tsv"
|
||||||
|
local total_scenarios
|
||||||
|
total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
|
||||||
|
log "generated ${total_scenarios} scenarios for ${config_label}"
|
||||||
|
|
||||||
|
# Write metadata.
|
||||||
|
ensure_result_root "$result_root"
|
||||||
|
write_metadata_json \
|
||||||
|
"${result_root}/results.json" \
|
||||||
|
"${EXPERIMENT_NAME}_${config_label}" \
|
||||||
|
"$RUN_ID" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"sglang" \
|
||||||
|
"sglang" \
|
||||||
|
"$HARDWARE" \
|
||||||
|
"$ACCELERATOR" \
|
||||||
|
"$CHIP" \
|
||||||
|
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||||
|
"$DOCKER_IMAGE" \
|
||||||
|
"H200 SGLang TP×DP matrix for DeepSeek-V4-Flash on RTX 6000D"
|
||||||
|
|
||||||
|
local server_args_str
|
||||||
|
server_args_str="$(build_server_args "$tp" "$dp")"
|
||||||
|
jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \
|
||||||
|
'.config = {
|
||||||
|
"tp": ($tp | tonumber),
|
||||||
|
"dp": ($dp | tonumber),
|
||||||
|
"cuda_visible_devices": $cuda,
|
||||||
|
"backend": "sglang",
|
||||||
|
"server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_sglang_dp.sh",
|
||||||
|
"server_args": $args
|
||||||
|
}' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \
|
||||||
|
mv "${result_root}/results.json.tmp" "${result_root}/results.json"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
log "DRY_RUN: would start server with args: ${server_args_str}"
|
||||||
|
local line
|
||||||
|
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
|
||||||
|
log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
|
||||||
|
done
|
||||||
|
log "===== ${config_label} DONE (dry run) ====="
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Initialize skipped_after_oom.csv for this run.
|
||||||
|
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||||
|
if [[ ! -f "$skipped_csv" ]]; then
|
||||||
|
echo "engine,tp,dp,mark,isl,dsl,concurrency,status,reason,detail_log" > "$skipped_csv"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start server once for this TP×DP config.
|
||||||
|
if ! start_server "$tp" "$dp"; then
|
||||||
|
log "ERROR: ${config_label} failed to start; skipping all scenarios"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp"
|
||||||
|
log "===== ${config_label} DONE ====="
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Warmup with a small prompt before the first scenario.
|
||||||
|
run_warmup 1024 128 || true
|
||||||
|
|
||||||
|
# Read scenarios into an array so we can skip remaining entries on failure.
|
||||||
|
local -a scenarios=()
|
||||||
|
while IFS= read -r line; do
|
||||||
|
scenarios+=("$line")
|
||||||
|
done < <(tail -n +2 "$scenario_tsv")
|
||||||
|
|
||||||
|
local i mark isl dsl conc num
|
||||||
|
local output_file detail_log gpu_csv sname bench_rc
|
||||||
|
for (( i = 0; i < ${#scenarios[@]}; i++ )); do
|
||||||
|
IFS=$'\t' read -r mark isl dsl conc num <<< "${scenarios[$i]}"
|
||||||
|
|
||||||
|
if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then
|
||||||
|
log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
sname="c${conc}_i${isl}_o${dsl}"
|
||||||
|
output_file="${raw_dir}/sglang_main_${conc}_${isl}_${dsl}.jsonl"
|
||||||
|
detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log"
|
||||||
|
gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${dsl}.csv"
|
||||||
|
|
||||||
|
if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then
|
||||||
|
log "skipping already-processed ${config_label} scenario: ${sname}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
|
||||||
|
|
||||||
|
local gpu_pid
|
||||||
|
gpu_pid="$(start_gpu_monitor "$gpu_csv")"
|
||||||
|
|
||||||
|
bench_rc=0
|
||||||
|
timeout "$SCENARIO_TIMEOUT_S" bash -c '
|
||||||
|
run_bench_serving \
|
||||||
|
--backend sglang \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port "'"$SGLANG_PORT"'" \
|
||||||
|
--random-input-len "'"$isl"'" \
|
||||||
|
--random-output-len "'"$dsl"'" \
|
||||||
|
--num-prompts "'"$num"'" \
|
||||||
|
--max-concurrency "'"$conc"'" \
|
||||||
|
--request-rate 10000 \
|
||||||
|
--output-file "'"$output_file"'" \
|
||||||
|
--output-details \
|
||||||
|
> "'"$detail_log"'" 2>&1
|
||||||
|
' || bench_rc=$?
|
||||||
|
|
||||||
|
stop_gpu_monitor "$gpu_pid"
|
||||||
|
|
||||||
|
if [[ "$bench_rc" -eq 0 ]]; then
|
||||||
|
log "finished ${config_label} scenario: output=${output_file}"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"completed\"" \
|
||||||
|
"note=\"benchmark finished successfully\""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Failure handling.
|
||||||
|
if detect_oom "$detail_log" "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log"; then
|
||||||
|
log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"OOM\"" \
|
||||||
|
"note=\"detected CUDA out-of-memory\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}"
|
||||||
|
if [[ "$mark" == "P" ]]; then
|
||||||
|
log "optional (P) scenario failed; recording as skipped and continuing"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"skipped_optional\"" \
|
||||||
|
"note=\"optional scenario failed (rc=${bench_rc})\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Mandatory scenario failed but not OOM: try to restart the server.
|
||||||
|
if restart_server "$tp" "$dp"; then
|
||||||
|
run_warmup 1024 128 || true
|
||||||
|
log "resuming ${config_label} after server restart"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset "$BENCH_DATASET_NAME" --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"FAILED\"" \
|
||||||
|
"note=\"scenario failed and server restart failed (rc=${bench_rc})\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "dsl=${dsl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
done
|
||||||
|
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
|
||||||
|
# Parse results.
|
||||||
|
log "parsing ${config_label} results"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \
|
||||||
|
>> "${phase_log_dir}/parse.log" 2>&1 || {
|
||||||
|
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
log "===== ${config_label} DONE ====="
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Cleanup any leftovers.
|
||||||
|
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||||
|
read -r tp dp <<< "$cfg"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Run each parallel configuration.
|
||||||
|
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||||
|
read -r tp dp <<< "$cfg"
|
||||||
|
run_parallel_config "$tp" "$dp"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Generate cross-configuration comparison.
|
||||||
|
log "generating comparison report"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/compare.py" \
|
||||||
|
--run-root "${RESULT_BASE}/${RUN_ID}" \
|
||||||
|
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
||||||
|
>> "${log_dir_global}/compare.log" 2>&1 || {
|
||||||
|
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|
||||||
16
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_dp.sh
Executable file
16
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_dp.sh
Executable file
@ -0,0 +1,16 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Entry point for the matrix runner (run_bench.sh calls this as
|
||||||
|
# start_sglang_dp.sh <tp> <dp>). For this multi-node experiment it always
|
||||||
|
# delegates to the multi-node orchestrator, which brings up both nodes.
|
||||||
|
#
|
||||||
|
# Usage: start_sglang_dp.sh <tp> <dp>
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1:?usage: start_sglang_dp.sh <tp> <dp>}"
|
||||||
|
DP="${2:?usage: start_sglang_dp.sh <tp> <dp>}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
# This experiment is multi-node only; there is no single-node venv path.
|
||||||
|
# USE_DOCKER is kept for config.env compatibility but ignored here.
|
||||||
|
exec "${SCRIPT_DIR}/start_sglang_multinode.sh" "$TP" "$DP"
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Multi-node orchestrator for GLM-5.2-FP8 TP=16 across pro6000D.1 + pro6000D.3,
|
||||||
|
# through the shared deployment layer (`python -m sskj.deploy start` with
|
||||||
|
# NODE_HOSTS orchestration). Rank 0 (local) is started without ssh.
|
||||||
|
#
|
||||||
|
# Usage: start_sglang_multinode.sh <tp> <dp>
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1:-${TP_SIZE:-16}}"
|
||||||
|
DP="${2:-${DP_SIZE:-1}}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
|
echo "=== Multi-node start: TP=${TP} DP=${DP} nnodes=${NNODES} via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
|
||||||
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
|
--tp "$TP" --dp "$DP" \
|
||||||
|
--port "${SGLANG_PORT:-30031}" \
|
||||||
|
--model-path "$MODEL_PATH" \
|
||||||
|
--log-dir "${RUNTIME_BASE}/logs" \
|
||||||
|
--dry-run
|
||||||
|
else
|
||||||
|
deploy_start_multinode \
|
||||||
|
"$DEPLOY_PROFILE" "$TP" "$DP" \
|
||||||
|
"${RUNTIME_BASE}/logs" \
|
||||||
|
"${SGLANG_PORT:-30031}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$EXPERIMENT"
|
||||||
|
fi
|
||||||
46
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_node.sh
Executable file
46
experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/start_sglang_node.sh
Executable file
@ -0,0 +1,46 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start one GLM-5.2-FP8 SGLang node through the shared deployment layer.
|
||||||
|
# Convenience for per-node debugging; the multinode orchestrator
|
||||||
|
# (start_sglang_multinode.sh) does not use this anymore.
|
||||||
|
#
|
||||||
|
# Usage: start_sglang_node.sh <node_rank>
|
||||||
|
set -e
|
||||||
|
|
||||||
|
NODE_RANK="${1:?usage: start_sglang_node.sh <node_rank>}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
|
echo "=== Start node${NODE_RANK} (TP=${TP_SIZE}, DP=${DP_SIZE}, nnodes=${NNODES}) via deploy profile ==="
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy start \
|
||||||
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
|
--tp "${TP_SIZE:-16}" --dp "${DP_SIZE:-1}" \
|
||||||
|
--port "${SGLANG_PORT:-30031}" \
|
||||||
|
--model-path "$MODEL_PATH" \
|
||||||
|
--node-rank "$NODE_RANK" \
|
||||||
|
--log-dir "${RUNTIME_BASE}/logs" \
|
||||||
|
--dry-run
|
||||||
|
else
|
||||||
|
deploy_start \
|
||||||
|
"$DEPLOY_PROFILE" "${TP_SIZE:-16}" "${DP_SIZE:-1}" \
|
||||||
|
"${RUNTIME_BASE}/logs" \
|
||||||
|
"${SGLANG_PORT:-30031}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$EXPERIMENT" \
|
||||||
|
"$NODE_RANK"
|
||||||
|
fi
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Tear down the multi-node GLM-5.2-FP8 deployment (both nodes) through the
|
||||||
|
# shared deployment layer.
|
||||||
|
#
|
||||||
|
# Usage: stop_sglang_multinode.sh <tp> <dp>
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1:-${TP_SIZE:-16}}"
|
||||||
|
DP="${2:-${DP_SIZE:-1}}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/glm52_pro6000_sglang_tp16}"
|
||||||
|
|
||||||
|
echo "=== Multi-node stop: TP=${TP} DP=${DP} via deploy profile ${DEPLOY_PROFILE} ==="
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON:-python3}" -m sskj.deploy stop \
|
||||||
|
--profile "$(deploy_profile_abs "$DEPLOY_PROFILE")" \
|
||||||
|
--tp "$TP" --dp "$DP" \
|
||||||
|
--port "${SGLANG_PORT:-30031}" \
|
||||||
|
--model-path "$MODEL_PATH" \
|
||||||
|
--container-name "$EXPERIMENT" \
|
||||||
|
--dry-run
|
||||||
|
else
|
||||||
|
deploy_stop \
|
||||||
|
"$DEPLOY_PROFILE" "$TP" "$DP" \
|
||||||
|
"${SGLANG_PORT:-30031}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$EXPERIMENT"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Local fallback: kill any stray launch_server for this model.
|
||||||
|
pkill -9 -f "sglang.launch_server.*${MODEL_NAME}" 2>/dev/null || true
|
||||||
|
pkill -9 -f "sglang.launch_server.*${MODEL_PATH}" 2>/dev/null || true
|
||||||
|
echo "=== both nodes stopped ==="
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
import math
|
||||||
|
import os
|
||||||
|
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from sglang.kernels.ops.attention.dsa.tilelang_kernel import (
|
||||||
|
sparse_mla_fwd_decode_combine,
|
||||||
|
sparse_mla_fwd_decode_partial,
|
||||||
|
tilelang_sparse_fwd,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
torch.manual_seed(0)
|
||||||
|
heads = int(os.environ.get("HEADS", "4"))
|
||||||
|
d_v = 256
|
||||||
|
d_tail = 64
|
||||||
|
topk = 2048
|
||||||
|
block_i = 32
|
||||||
|
threads = 128
|
||||||
|
scale = 1.0 / math.sqrt(d_v + d_tail)
|
||||||
|
|
||||||
|
q = torch.randn((1, heads, d_v + d_tail), device="cuda", dtype=torch.bfloat16)
|
||||||
|
kv = torch.randn(
|
||||||
|
(topk, 1, d_v + d_tail), device="cuda", dtype=torch.bfloat16
|
||||||
|
)
|
||||||
|
indices = torch.arange(topk, device="cuda", dtype=torch.int32).view(1, 1, topk)
|
||||||
|
|
||||||
|
if os.environ.get("USE_WRAPPER") == "1":
|
||||||
|
output = tilelang_sparse_fwd(q, kv, indices, scale, d_v)
|
||||||
|
else:
|
||||||
|
partial_kernel = sparse_mla_fwd_decode_partial(
|
||||||
|
heads,
|
||||||
|
d_v,
|
||||||
|
d_tail,
|
||||||
|
topk,
|
||||||
|
sm_scale=scale,
|
||||||
|
block_I=block_i,
|
||||||
|
inner_iter=1,
|
||||||
|
num_stages=1,
|
||||||
|
threads=threads,
|
||||||
|
)
|
||||||
|
partial_o, partial_lse = partial_kernel(
|
||||||
|
q.unsqueeze(0), kv.unsqueeze(0), indices.unsqueeze(0)
|
||||||
|
)
|
||||||
|
groups = topk // block_i
|
||||||
|
combine_kernel = sparse_mla_fwd_decode_combine(
|
||||||
|
heads,
|
||||||
|
d_v,
|
||||||
|
groups * block_i,
|
||||||
|
head_per_block=4,
|
||||||
|
block_I=block_i,
|
||||||
|
threads=threads,
|
||||||
|
)
|
||||||
|
output = combine_kernel(partial_o, partial_lse)
|
||||||
|
|
||||||
|
q_float = q[0].float()
|
||||||
|
k_float = kv[:, 0].float()
|
||||||
|
scores = q_float @ k_float.transpose(0, 1) * scale
|
||||||
|
probs = torch.softmax(scores, dim=-1)
|
||||||
|
reference = probs @ k_float[:, :d_v]
|
||||||
|
actual = output.reshape(heads, d_v).float()
|
||||||
|
error = (actual - reference).abs()
|
||||||
|
|
||||||
|
print(f"output_shape={tuple(output.shape)}")
|
||||||
|
print(f"max_abs_error={error.max().item():.6f}")
|
||||||
|
print(f"mean_abs_error={error.mean().item():.6f}")
|
||||||
|
print(f"all_finite={torch.isfinite(actual).all().item()}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
File diff suppressed because it is too large
Load Diff
@ -17,6 +17,8 @@ DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_clean
|
|||||||
|
|
||||||
# Serving port + container.
|
# Serving port + container.
|
||||||
PORT="${PORT:-30010}"
|
PORT="${PORT:-30010}"
|
||||||
|
# Alias for the unified bench layer (`python -m sskj.bench`).
|
||||||
|
SGLANG_PORT="${SGLANG_PORT:-${PORT:-30010}}"
|
||||||
CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_pro6000_sglang_tp8}"
|
CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_pro6000_sglang_tp8}"
|
||||||
|
|
||||||
# Docker image: sm120-capable SGLang (built/verified on this RTX 6000D box),
|
# Docker image: sm120-capable SGLang (built/verified on this RTX 6000D box),
|
||||||
|
|||||||
@ -85,14 +85,8 @@ fi
|
|||||||
|
|
||||||
stop_server() {
|
stop_server() {
|
||||||
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
||||||
log "stopping server (kill docker run PID -> --rm removes container)"
|
log "stopping server via deploy profile"
|
||||||
if [[ -f "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid" ]]; then
|
bash "${SCRIPT_DIR}/stop_server.sh" >> "${LOG_DIR}/stop_server.outer.log" 2>&1 || true
|
||||||
local pid
|
|
||||||
pid="$(cat "${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid")"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 3
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,92 +1,31 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Start the SGLang server for Qwen3-235B-A22B on NVIDIA RTX 6000D (TP=8, 8 GPUs).
|
# Start the Qwen3-235B-A22B SGLang TP=8 server through the shared deployment layer.
|
||||||
#
|
|
||||||
# NVIDIA counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8/start_server.sh.
|
|
||||||
# Uses the standard sglang qwen3_moe path (no Kunlun patch, no XPU env).
|
|
||||||
# Launch pattern mirrors pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh.
|
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
# Source config before platform so our CONTAINER_NAME/PORT defaults win.
|
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/config.env"
|
source "${SCRIPT_DIR}/config.env"
|
||||||
# shellcheck source=/dev/null
|
# shellcheck source=/dev/null
|
||||||
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
||||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
|
||||||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
LOG_DIR="${RESULT_ROOT}/logs"
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
mkdir -p "$LOG_DIR" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
|
|
||||||
|
|
||||||
log_init "${LOG_DIR}/start_server.log"
|
log "starting server for ${EXPERIMENT_NAME} (NVIDIA, TP=${TP}) via deploy profile"
|
||||||
log "starting server for ${EXPERIMENT_NAME} (NVIDIA, TP=${TP})"
|
|
||||||
log "model: ${MODEL_PATH}"
|
log "model: ${MODEL_PATH}"
|
||||||
log "image: ${DOCKER_IMAGE}"
|
log "image: ${DOCKER_IMAGE}"
|
||||||
log "port: ${PORT} container: ${CONTAINER_NAME}"
|
log "port: ${PORT} container: ${CONTAINER_NAME}"
|
||||||
|
|
||||||
SERVER_LOG="${LOG_DIR}/server.log"
|
deploy_start \
|
||||||
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT_NAME}.pid"
|
"${DEPLOY_PROFILE:-pro6000/qwen3_235b_pro6000_sglang_tp8}" \
|
||||||
rm -f "$PID_FILE"
|
"${TP:-8}" "1" \
|
||||||
|
"${RUNTIME_BASE}/logs" \
|
||||||
# Remove any stale container with the same name.
|
"${PORT:-30010}" \
|
||||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
"$MODEL_PATH" \
|
||||||
|
"$CONTAINER_NAME"
|
||||||
# Standard sglang launch for qwen3_moe on NVIDIA (bf16 auto, flashinfer attn).
|
|
||||||
# Mount /data so model + experiment results dir + dataset are all at the same
|
|
||||||
# path inside the container (bench client writes results there directly).
|
|
||||||
DTYPE_ARGS=()
|
|
||||||
if [[ "${DTYPE}" != "auto" ]]; then
|
|
||||||
DTYPE_ARGS=(--dtype "${DTYPE}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
nohup docker run --rm \
|
|
||||||
--name "$CONTAINER_NAME" \
|
|
||||||
--gpus all \
|
|
||||||
--privileged \
|
|
||||||
--ipc=host \
|
|
||||||
--network host \
|
|
||||||
--ulimit memlock=-1 \
|
|
||||||
--ulimit stack=67108864 \
|
|
||||||
--entrypoint python3 \
|
|
||||||
-v /data:/data \
|
|
||||||
-v "${CACHE_DIR}:/root/.cache" \
|
|
||||||
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
|
||||||
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
|
||||||
-e PYTHONUNBUFFERED=1 \
|
|
||||||
-e HF_HUB_OFFLINE=1 \
|
|
||||||
-e TRANSFORMERS_OFFLINE=1 \
|
|
||||||
-e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
|
||||||
"$DOCKER_IMAGE" \
|
|
||||||
-m sglang.launch_server \
|
|
||||||
--model-path "${MODEL_PATH}" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--tp-size "${TP}" \
|
|
||||||
--mem-fraction-static "${MEM_FRACTION_STATIC}" \
|
|
||||||
--context-length "${CONTEXT_LENGTH}" \
|
|
||||||
--max-running-requests "${MAX_RUNNING_REQUESTS}" \
|
|
||||||
--host 0.0.0.0 \
|
|
||||||
--port "${PORT}" \
|
|
||||||
"${DTYPE_ARGS[@]}" \
|
|
||||||
> "$SERVER_LOG" 2>&1 &
|
|
||||||
|
|
||||||
PID=$!
|
|
||||||
echo "$PID" > "$PID_FILE"
|
|
||||||
log "PID: ${PID}"
|
|
||||||
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-1200}"
|
|
||||||
log "waiting for health on http://127.0.0.1:${PORT}/health (timeout=${HEALTH_TIMEOUT}s)"
|
|
||||||
|
|
||||||
if health_check 127.0.0.1 "$PORT" "$HEALTH_TIMEOUT"; then
|
|
||||||
log "sglang server is READY at http://127.0.0.1:${PORT}"
|
|
||||||
log "server log: ${SERVER_LOG} (container: ${CONTAINER_NAME})"
|
|
||||||
exit 0
|
|
||||||
else
|
|
||||||
log "ERROR: server not healthy after ${HEALTH_TIMEOUT}s"
|
|
||||||
log "----- last 200 lines of server log -----"
|
|
||||||
tail -200 "$SERVER_LOG" 2>/dev/null || docker logs --tail 200 "$CONTAINER_NAME" 2>&1 | tail -200
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|||||||
23
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/stop_server.sh
Executable file
23
experiments/pro6000/qwen3_235b_pro6000_sglang_tp8/stop_server.sh
Executable file
@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Stop the Qwen3-235B-A22B SGLang server through the shared deployment layer.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/deploy_cli.sh"
|
||||||
|
|
||||||
|
log "stopping server for ${EXPERIMENT_NAME} via deploy profile"
|
||||||
|
deploy_stop \
|
||||||
|
"${DEPLOY_PROFILE:-pro6000/qwen3_235b_pro6000_sglang_tp8}" \
|
||||||
|
"${TP:-8}" "1" \
|
||||||
|
"${PORT:-30010}" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"$CONTAINER_NAME"
|
||||||
@ -80,7 +80,48 @@ experiments/<platform>/<experiment>/results/<run_id>/
|
|||||||
跨机测试时可用 `--client docker --client-image lmsysorg/sglang:latest`
|
跨机测试时可用 `--client docker --client-image lmsysorg/sglang:latest`
|
||||||
指定外部 client。
|
指定外部 client。
|
||||||
|
|
||||||
## 7. 常见问题
|
## 7. pro6000（RTX 6000D）平台说明
|
||||||
|
|
||||||
|
实验目录 `experiments/pro6000/`，部署 profile 见 `deploy/profiles/pro6000/`。
|
||||||
|
|
||||||
|
可测实验：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHONPATH=src python3 -m sskj.bench list --platform pro6000
|
||||||
|
# dsv4_pro6000_sglang_tp16 / dsv4_pro6000_sglang_tp16_eagle
|
||||||
|
# dsv4_pro6000_sglang_tp_dp_matrix / dsv4_pro6000_vllm_tp_dp_matrix
|
||||||
|
# glm52_pro6000_sglang_multinode_tp16 / qwen3_235b_pro6000_sglang_tp8
|
||||||
|
```
|
||||||
|
|
||||||
|
单节点实验（dsv4/vllm tp_dp_matrix、qwen3，均 8x RTX 6000D 单机）：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHONPATH=src python3 -m sskj.bench run \
|
||||||
|
--url http://<head-ip>:30031 \
|
||||||
|
--platform pro6000 \
|
||||||
|
--experiment dsv4_pro6000_sglang_tp_dp_matrix \
|
||||||
|
--profile smoke
|
||||||
|
```
|
||||||
|
|
||||||
|
多节点实验（tp16 / tp16_eagle / glm52，2 台机器 16x RTX 6000D）：
|
||||||
|
- head 节点 pro6000D.1（`10.101.0.11` / `174.1.51.5`，HTTP API 所在）
|
||||||
|
- worker 节点 pro6000D.3（`10.101.0.13` / `174.1.51.7`，纯计算）
|
||||||
|
- `--url` 固定指向 head 节点端口（tp16/eagle: 30000，glm52: 30031），
|
||||||
|
bench client 在 head 节点跑，worker 由部署层通过 ssh 管理：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
PYTHONPATH=src python3 -m sskj.bench run \
|
||||||
|
--url http://10.101.0.11:30000 \
|
||||||
|
--platform pro6000 \
|
||||||
|
--experiment dsv4_pro6000_sglang_tp16 \
|
||||||
|
--profile smoke
|
||||||
|
```
|
||||||
|
|
||||||
|
vLLM 实验（`dsv4_pro6000_vllm_tp_dp_matrix`）的 bench client 使用 SGLang
|
||||||
|
镜像（vLLM 镜像不含 bench_serving），由 `DOCKER_CLIENT_IMAGE` 自动选择，
|
||||||
|
无需额外参数。
|
||||||
|
|
||||||
|
## 8. 常见问题
|
||||||
|
|
||||||
- 服务不健康：确认 `--url` 的端口可从测试机访问，服务已启动且 `/health` 可通。
|
- 服务不健康：确认 `--url` 的端口可从测试机访问，服务已启动且 `/health` 可通。
|
||||||
- 数据集缺失：仓库 `datasets/` 下没有 ShareGPT 文件时，统一 CLI 会自动退回
|
- 数据集缺失：仓库 `datasets/` 下没有 ShareGPT 文件时，统一 CLI 会自动退回
|
||||||
|
|||||||
@ -19,6 +19,11 @@ deploy_start() {
|
|||||||
local port="$5"
|
local port="$5"
|
||||||
local model_path="$6"
|
local model_path="$6"
|
||||||
local container="$7"
|
local container="$7"
|
||||||
|
local node_rank="${8:-}"
|
||||||
|
local -a extra=()
|
||||||
|
if [[ -n "$node_rank" ]]; then
|
||||||
|
extra+=(--node-rank "$node_rank")
|
||||||
|
fi
|
||||||
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy start \
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy start \
|
||||||
--profile "$(deploy_profile_abs "$profile")" \
|
--profile "$(deploy_profile_abs "$profile")" \
|
||||||
--tp "$tp" \
|
--tp "$tp" \
|
||||||
@ -26,7 +31,8 @@ deploy_start() {
|
|||||||
--log-dir "$log_dir" \
|
--log-dir "$log_dir" \
|
||||||
--port "$port" \
|
--port "$port" \
|
||||||
--model-path "$model_path" \
|
--model-path "$model_path" \
|
||||||
--container-name "$container"
|
--container-name "$container" \
|
||||||
|
"${extra[@]}"
|
||||||
}
|
}
|
||||||
|
|
||||||
deploy_render_args() {
|
deploy_render_args() {
|
||||||
@ -42,3 +48,63 @@ deploy_render_args() {
|
|||||||
--port "$port" \
|
--port "$port" \
|
||||||
--model-path "$model_path"
|
--model-path "$model_path"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Multi-node convenience helpers: orchestrate all NODE_HOSTS from one host.
|
||||||
|
|
||||||
|
deploy_start_multinode() {
|
||||||
|
local profile="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local log_dir="$4"
|
||||||
|
local port="$5"
|
||||||
|
local model_path="$6"
|
||||||
|
local container="$7"
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy start \
|
||||||
|
--profile "$(deploy_profile_abs "$profile")" \
|
||||||
|
--tp "$tp" \
|
||||||
|
--dp "$dp" \
|
||||||
|
--log-dir "$log_dir" \
|
||||||
|
--port "$port" \
|
||||||
|
--model-path "$model_path" \
|
||||||
|
--container-name "$container"
|
||||||
|
}
|
||||||
|
|
||||||
|
deploy_stop() {
|
||||||
|
local profile="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local port="$4"
|
||||||
|
local model_path="$5"
|
||||||
|
local container="$6"
|
||||||
|
local node_rank="${7:-}"
|
||||||
|
local -a extra=()
|
||||||
|
if [[ -n "$node_rank" ]]; then
|
||||||
|
extra+=(--node-rank "$node_rank")
|
||||||
|
fi
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy stop \
|
||||||
|
--profile "$(deploy_profile_abs "$profile")" \
|
||||||
|
--tp "$tp" \
|
||||||
|
--dp "$dp" \
|
||||||
|
--port "$port" \
|
||||||
|
--model-path "$model_path" \
|
||||||
|
--container-name "$container" \
|
||||||
|
"${extra[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
deploy_status() {
|
||||||
|
local profile="$1"
|
||||||
|
local tp="${2:-}"
|
||||||
|
local dp="${3:-}"
|
||||||
|
local port="${4:-}"
|
||||||
|
local model_path="${5:-}"
|
||||||
|
local container="${6:-}"
|
||||||
|
local -a cmd=(
|
||||||
|
--profile "$(deploy_profile_abs "$profile")"
|
||||||
|
)
|
||||||
|
[[ -n "$tp" ]] && cmd+=(--tp "$tp")
|
||||||
|
[[ -n "$dp" ]] && cmd+=(--dp "$dp")
|
||||||
|
[[ -n "$port" ]] && cmd+=(--port "$port")
|
||||||
|
[[ -n "$model_path" ]] && cmd+=(--model-path "$model_path")
|
||||||
|
[[ -n "$container" ]] && cmd+=(--container-name "$container")
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" "${DEPLOY_PYTHON}" -m sskj.deploy status "${cmd[@]}"
|
||||||
|
}
|
||||||
|
|||||||
@ -3,7 +3,8 @@
|
|||||||
The repository keeps deployment and platform configuration in shell `.env`
|
The repository keeps deployment and platform configuration in shell `.env`
|
||||||
files so existing bash scripts can still source them. This parser understands
|
files so existing bash scripts can still source them. This parser understands
|
||||||
the scalar subset those files use: `KEY=value`, `export KEY=value`, quoted
|
the scalar subset those files use: `KEY=value`, `export KEY=value`, quoted
|
||||||
values, comments, and `${VAR:-default}` substitution.
|
values, comments, and `${VAR:-default}` substitution. Nested defaults such as
|
||||||
|
`${A:-${B}/path}` are supported via balanced-brace scanning.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@ -13,7 +14,7 @@ import re
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
_ASSIGN_RE = re.compile(r"^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$")
|
_ASSIGN_RE = re.compile(r"^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$")
|
||||||
_VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
|
_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||||
|
|
||||||
|
|
||||||
def _strip_quotes(value: str) -> str:
|
def _strip_quotes(value: str) -> str:
|
||||||
@ -42,20 +43,61 @@ def _strip_inline_comment(value: str) -> str:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _find_brace_end(value: str, start: int) -> int | None:
|
||||||
|
"""Index of the `}` closing the `${` at `start`, handling nested `${...}`."""
|
||||||
|
depth = 0
|
||||||
|
i = start
|
||||||
|
n = len(value)
|
||||||
|
while i < n:
|
||||||
|
if value.startswith("${", i):
|
||||||
|
depth += 1
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
if value[i] == "}":
|
||||||
|
depth -= 1
|
||||||
|
if depth == 0:
|
||||||
|
return i
|
||||||
|
i += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _split_default(inner: str) -> tuple[str, str | None]:
|
||||||
|
"""Split `NAME` or `NAME:-default` (default may itself contain `${...}`)."""
|
||||||
|
for sep in (":-", "-"):
|
||||||
|
idx = inner.find(sep)
|
||||||
|
if idx > 0 and _NAME_RE.match(inner[:idx]):
|
||||||
|
return inner[:idx], inner[idx + len(sep):]
|
||||||
|
if _NAME_RE.match(inner):
|
||||||
|
return inner, None
|
||||||
|
return inner, None
|
||||||
|
|
||||||
|
|
||||||
def _expand(value: str, env: dict[str, str], depth: int = 0) -> str:
|
def _expand(value: str, env: dict[str, str], depth: int = 0) -> str:
|
||||||
if depth > 6:
|
if depth > 6:
|
||||||
return value
|
return value
|
||||||
|
|
||||||
def repl(match: re.Match[str]) -> str:
|
out: list[str] = []
|
||||||
name = match.group(1)
|
i = 0
|
||||||
default = match.group(2)
|
n = len(value)
|
||||||
if name in env and env[name] != "":
|
while i < n:
|
||||||
return env[name]
|
if value.startswith("${", i):
|
||||||
if default is not None:
|
end = _find_brace_end(value, i)
|
||||||
return _expand(default, env, depth + 1)
|
if end is None:
|
||||||
return match.group(0)
|
out.append(value[i:])
|
||||||
|
break
|
||||||
return _VAR_RE.sub(repl, value)
|
inner = value[i + 2:end]
|
||||||
|
name, default = _split_default(inner)
|
||||||
|
if _NAME_RE.match(name) and name in env and env[name] != "":
|
||||||
|
out.append(env[name])
|
||||||
|
elif _NAME_RE.match(name) and default is not None:
|
||||||
|
out.append(_expand(default, env, depth + 1))
|
||||||
|
else:
|
||||||
|
out.append(value[i:end + 1])
|
||||||
|
i = end + 1
|
||||||
|
continue
|
||||||
|
out.append(value[i])
|
||||||
|
i += 1
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
def parse_env_file(
|
def parse_env_file(
|
||||||
|
|||||||
@ -1,14 +1,33 @@
|
|||||||
"""Docker/native server lifecycle for deployment profiles."""
|
"""Docker/native server lifecycle for deployment profiles.
|
||||||
|
|
||||||
|
Supports single-node and multi-node (NODE_HOSTS) docker deployments. For
|
||||||
|
multi-node profiles, start/stop/status distribute docker commands to each
|
||||||
|
node over ssh; health checks only target the head (rank 0) node.
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from sskj.bench.runner import split_env_assignments, wait_health
|
from sskj.bench.runner import split_env_assignments, wait_health
|
||||||
|
|
||||||
|
_NODE_RANK_RE = re.compile(r"\$\{NODE_RANK(?::-([^}]*))?\}")
|
||||||
|
_TEMPLATED_KEYS = (
|
||||||
|
"CONTAINER_NAME",
|
||||||
|
"LAUNCH_ARGS",
|
||||||
|
"DEVICE_VARS",
|
||||||
|
"ENGINE_ENV",
|
||||||
|
"DOCKER_FLAGS",
|
||||||
|
"VOLUMES",
|
||||||
|
"PATCH_MOUNTS",
|
||||||
|
"BOOTSTRAP",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _log_dir(root: Path, log_dir: str | None) -> Path:
|
def _log_dir(root: Path, log_dir: str | None) -> Path:
|
||||||
path = Path(log_dir) if log_dir else root / "deploy" / "logs"
|
path = Path(log_dir) if log_dir else root / "deploy" / "logs"
|
||||||
@ -21,21 +40,50 @@ def _health_url(profile: dict[str, str]) -> str:
|
|||||||
return f"http://{host}:{profile['PORT']}"
|
return f"http://{host}:{profile['PORT']}"
|
||||||
|
|
||||||
|
|
||||||
def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: str | None = None) -> int:
|
def _node_hosts(profile: dict[str, str]) -> list[str]:
|
||||||
validate_runtime(profile)
|
return [h for h in shlex.split(profile.get("NODE_HOSTS", "")) if h]
|
||||||
logs = _log_dir(root, log_dir)
|
|
||||||
runtime = profile["RUNTIME"]
|
|
||||||
container = profile.get("CONTAINER_NAME", "")
|
|
||||||
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
|
|
||||||
|
|
||||||
if runtime == "docker":
|
|
||||||
if not profile.get("DOCKER_IMAGE"):
|
|
||||||
raise SystemExit("docker runtime requires DOCKER_IMAGE")
|
|
||||||
if not container:
|
|
||||||
raise SystemExit("docker runtime requires CONTAINER_NAME")
|
|
||||||
if not dry_run:
|
|
||||||
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
|
||||||
|
|
||||||
|
def _ssh_host(profile: dict[str, str], host: str) -> str:
|
||||||
|
user = profile.get("NODE_SSH_USER", "root")
|
||||||
|
if not user or "@" in host:
|
||||||
|
return host
|
||||||
|
return f"{user}@{host}"
|
||||||
|
|
||||||
|
|
||||||
|
def _sub_node_rank(profile: dict[str, str], rank: int) -> dict[str, str]:
|
||||||
|
"""Return a profile copy with NODE_RANK rendered for the given rank."""
|
||||||
|
rendered = dict(profile)
|
||||||
|
container_name = str(rendered.get("CONTAINER_NAME", "server"))
|
||||||
|
has_template = _NODE_RANK_RE.search(container_name) is not None
|
||||||
|
|
||||||
|
def _sub(value: str | None) -> str:
|
||||||
|
if not value:
|
||||||
|
return value or ""
|
||||||
|
return _NODE_RANK_RE.sub(lambda m: m.group(1) or str(rank), str(value))
|
||||||
|
|
||||||
|
for key in _TEMPLATED_KEYS:
|
||||||
|
if key in rendered:
|
||||||
|
rendered[key] = _sub(rendered[key])
|
||||||
|
if not has_template:
|
||||||
|
rendered["CONTAINER_NAME"] = f"{container_name}_node{rank}"
|
||||||
|
rendered["NODE_RANK"] = str(rank)
|
||||||
|
return rendered
|
||||||
|
|
||||||
|
|
||||||
|
def _server_cmd(profile: dict[str, str]) -> str:
|
||||||
|
"""Launch command prefix (without LAUNCH_ARGS)."""
|
||||||
|
if profile.get("SERVER_CMD"):
|
||||||
|
return profile["SERVER_CMD"]
|
||||||
|
engine = (profile.get("ENGINE") or "sglang").lower()
|
||||||
|
python_bin = profile.get("CONTAINER_PYTHON", "python")
|
||||||
|
if engine == "vllm":
|
||||||
|
return f"{python_bin} -m vllm.entrypoints.openai.api_server"
|
||||||
|
return f"{python_bin} -m sglang.launch_server"
|
||||||
|
|
||||||
|
|
||||||
|
def _docker_run_cmd(profile: dict[str, str]) -> list[str]:
|
||||||
|
container = profile["CONTAINER_NAME"]
|
||||||
cmd = ["docker", "run", "-d", "--name", container]
|
cmd = ["docker", "run", "-d", "--name", container]
|
||||||
if profile.get("DOCKER_FLAGS"):
|
if profile.get("DOCKER_FLAGS"):
|
||||||
cmd += shlex.split(profile["DOCKER_FLAGS"])
|
cmd += shlex.split(profile["DOCKER_FLAGS"])
|
||||||
@ -54,18 +102,134 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
|
|||||||
encoded = base64.b64encode(bootstrap.encode("utf-8")).decode("ascii")
|
encoded = base64.b64encode(bootstrap.encode("utf-8")).decode("ascii")
|
||||||
cmd += ["bash", "-c", f"echo {encoded} | base64 -d | bash"]
|
cmd += ["bash", "-c", f"echo {encoded} | base64 -d | bash"]
|
||||||
else:
|
else:
|
||||||
python_bin = profile.get("CONTAINER_PYTHON", "python")
|
cmd += ["bash", "-c", f"{_server_cmd(profile)} {profile['LAUNCH_ARGS']}"]
|
||||||
cmd += ["bash", "-c", f"{python_bin} -m sglang.launch_server {profile['LAUNCH_ARGS']}"]
|
return cmd
|
||||||
|
|
||||||
(logs / f"{container}.cmd.txt").write_text(shlex.join(cmd) + "\n", encoding="utf-8")
|
|
||||||
if dry_run:
|
def _is_multinode(profile: dict[str, str], node_rank: str | None) -> bool:
|
||||||
print(shlex.join(cmd))
|
return len(_node_hosts(profile)) > 1 and node_rank is None
|
||||||
|
|
||||||
|
|
||||||
|
def _local_rank(profile: dict[str, str]) -> int:
|
||||||
|
"""Rank that runs on this host (no ssh); defaults to 0 (head)."""
|
||||||
|
try:
|
||||||
|
return int(profile.get("LOCAL_NODE_RANK", "0"))
|
||||||
|
except ValueError:
|
||||||
return 0
|
return 0
|
||||||
print(f"starting container {container} ...")
|
|
||||||
|
|
||||||
|
def _node_target(profile: dict[str, str], rank: int) -> str | None:
|
||||||
|
"""ssh host for the rank, or None when the rank runs locally."""
|
||||||
|
if rank == _local_rank(profile):
|
||||||
|
return None
|
||||||
|
return _ssh_host(profile, _node_hosts(profile)[rank])
|
||||||
|
|
||||||
|
|
||||||
|
def _run_docker_cmd(
|
||||||
|
profile: dict[str, str],
|
||||||
|
cmd: list[str],
|
||||||
|
host: str | None,
|
||||||
|
dry_run: bool,
|
||||||
|
log_path: Path | None = None,
|
||||||
|
) -> int:
|
||||||
|
joined = shlex.join(cmd)
|
||||||
|
if log_path is not None:
|
||||||
|
log_path.write_text(joined + "\n", encoding="utf-8")
|
||||||
|
if dry_run:
|
||||||
|
print(joined)
|
||||||
|
return 0
|
||||||
|
if host:
|
||||||
|
remote = f"bash -c {shlex.quote(joined)}"
|
||||||
|
result = subprocess.run(
|
||||||
|
["ssh", "-o", "StrictHostKeyChecking=no", host, remote],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(result.stdout, result.stderr)
|
||||||
|
return result.returncode
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
print(result.stdout, result.stderr)
|
print(result.stdout, result.stderr)
|
||||||
return result.returncode
|
return result.returncode
|
||||||
|
|
||||||
|
|
||||||
|
def _start_docker(
|
||||||
|
profile: dict[str, str],
|
||||||
|
root: Path,
|
||||||
|
dry_run: bool,
|
||||||
|
logs: Path,
|
||||||
|
) -> int:
|
||||||
|
if not profile.get("DOCKER_IMAGE"):
|
||||||
|
raise SystemExit("docker runtime requires DOCKER_IMAGE")
|
||||||
|
if not profile.get("CONTAINER_NAME"):
|
||||||
|
raise SystemExit("docker runtime requires CONTAINER_NAME")
|
||||||
|
|
||||||
|
hosts = _node_hosts(profile)
|
||||||
|
node_rank = profile.get("NODE_RANK")
|
||||||
|
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
|
||||||
|
|
||||||
|
if _is_multinode(profile, node_rank):
|
||||||
|
# Workers first (rank > 0), then head (rank 0), mirroring the legacy
|
||||||
|
# multi-node scripts so NCCL bootstrap finds the head up. The
|
||||||
|
# LOCAL_NODE_RANK rank runs locally (no ssh).
|
||||||
|
order = [r for r in range(len(hosts)) if r != 0] + [0]
|
||||||
|
for rank in order:
|
||||||
|
host = hosts[rank]
|
||||||
|
target = _node_target(profile, rank)
|
||||||
|
node_profile = _sub_node_rank(profile, rank)
|
||||||
|
cmd = _docker_run_cmd(node_profile)
|
||||||
|
if not dry_run:
|
||||||
|
if target:
|
||||||
|
subprocess.run(
|
||||||
|
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker rm -f {shlex.quote(node_profile['CONTAINER_NAME'])}"],
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "-f", node_profile["CONTAINER_NAME"]],
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
print(f"starting container {node_profile['CONTAINER_NAME']} on {host} (rank {rank}) ...")
|
||||||
|
rc = _run_docker_cmd(
|
||||||
|
profile,
|
||||||
|
cmd,
|
||||||
|
target,
|
||||||
|
dry_run,
|
||||||
|
logs / f"{node_profile['CONTAINER_NAME']}.cmd.txt",
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
return rc
|
||||||
|
if rank != 0:
|
||||||
|
print(" sleeping 5s before head ...")
|
||||||
|
if not dry_run:
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
head_url = f"http://{profile.get('HEALTH_HOST', hosts[0])}:{profile['PORT']}"
|
||||||
|
if dry_run:
|
||||||
|
print(f"[dry-run] health: {head_url}{profile.get('HEALTH_PATH', '/health')}")
|
||||||
|
return 0
|
||||||
|
if wait_health(head_url, profile.get("HEALTH_PATH", "/health"), health_wait):
|
||||||
|
print(f"server on {hosts[0]} is healthy")
|
||||||
|
return 0
|
||||||
|
print("ERROR: head failed health check; inspect per-node logs")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
# Single-node or explicit --node-rank: run locally on this host.
|
||||||
|
if not dry_run:
|
||||||
|
subprocess.run(["docker", "rm", "-f", profile["CONTAINER_NAME"]], capture_output=True, check=False)
|
||||||
|
|
||||||
|
cmd = _docker_run_cmd(profile)
|
||||||
|
container = profile["CONTAINER_NAME"]
|
||||||
|
print(f"starting container {container} ...")
|
||||||
|
rc = _run_docker_cmd(profile, cmd, None, dry_run, logs / f"{container}.cmd.txt")
|
||||||
|
if rc != 0:
|
||||||
|
return rc
|
||||||
|
if node_rank is not None and node_rank != "0":
|
||||||
|
print(f"node rank {node_rank} started; skipping health check")
|
||||||
|
return 0
|
||||||
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
|
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
|
||||||
print(f"container {container} is healthy")
|
print(f"container {container} is healthy")
|
||||||
return 0
|
return 0
|
||||||
@ -73,10 +237,21 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
|
|||||||
print(f"ERROR: container {container} failed health check")
|
print(f"ERROR: container {container} failed health check")
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: str | None = None) -> int:
|
||||||
|
validate_runtime(profile)
|
||||||
|
logs = _log_dir(root, log_dir)
|
||||||
|
if profile["RUNTIME"] == "docker":
|
||||||
|
return _start_docker(profile, root, dry_run, logs)
|
||||||
|
|
||||||
# native runtime
|
# native runtime
|
||||||
python_bin = profile.get("PYTHON_BIN") or profile.get("CONTAINER_PYTHON") or "python3"
|
python_bin = profile.get("PYTHON_BIN") or profile.get("CONTAINER_PYTHON") or "python3"
|
||||||
launch_args = shlex.split(profile.get("LAUNCH_ARGS", ""))
|
if profile.get("SERVER_CMD"):
|
||||||
cmd = [python_bin, "-m", "sglang.launch_server", *launch_args]
|
cmd = shlex.split(profile["SERVER_CMD"]) + shlex.split(profile.get("LAUNCH_ARGS", ""))
|
||||||
|
else:
|
||||||
|
engine = (profile.get("ENGINE") or "sglang").lower()
|
||||||
|
module = "vllm.entrypoints.openai.api_server" if engine == "vllm" else "sglang.launch_server"
|
||||||
|
cmd = [python_bin, "-m", module, *shlex.split(profile.get("LAUNCH_ARGS", ""))]
|
||||||
log_path = logs / f"{profile.get('MODEL_NAME', 'server')}.log"
|
log_path = logs / f"{profile.get('MODEL_NAME', 'server')}.log"
|
||||||
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
||||||
if dry_run:
|
if dry_run:
|
||||||
@ -86,6 +261,7 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
|
|||||||
proc = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT)
|
proc = subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT)
|
||||||
pid_file.write_text(str(proc.pid), encoding="utf-8")
|
pid_file.write_text(str(proc.pid), encoding="utf-8")
|
||||||
print(f"started native server pid={proc.pid} log={log_path}")
|
print(f"started native server pid={proc.pid} log={log_path}")
|
||||||
|
health_wait = int(profile.get("HEALTH_WAIT_S", "600") or 600)
|
||||||
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
|
if wait_health(_health_url(profile), profile.get("HEALTH_PATH", "/health"), health_wait):
|
||||||
print("native server is healthy")
|
print("native server is healthy")
|
||||||
return 0
|
return 0
|
||||||
@ -93,15 +269,45 @@ def start(profile: dict[str, str], root: Path, dry_run: bool = False, log_dir: s
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
def stop(profile: dict[str, str], root: Path) -> int:
|
def _docker_stop(profile: dict[str, str], dry_run: bool = False) -> int:
|
||||||
validate_runtime(profile)
|
node_rank = profile.get("NODE_RANK")
|
||||||
if profile["RUNTIME"] == "docker":
|
if _is_multinode(profile, node_rank):
|
||||||
|
for rank, host in enumerate(_node_hosts(profile)):
|
||||||
|
node_profile = _sub_node_rank(profile, rank)
|
||||||
|
name = node_profile["CONTAINER_NAME"]
|
||||||
|
target = _node_target(profile, rank)
|
||||||
|
if dry_run:
|
||||||
|
if target:
|
||||||
|
print(f"[dry-run] ssh {target} docker rm -f {name}")
|
||||||
|
else:
|
||||||
|
print(f"[dry-run] docker rm -f {name} (local)")
|
||||||
|
continue
|
||||||
|
if target:
|
||||||
|
subprocess.run(
|
||||||
|
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker rm -f {shlex.quote(name)}"],
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
print(f"container {name} removed on {host}")
|
||||||
|
else:
|
||||||
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
|
||||||
|
print(f"container {name} removed (local)")
|
||||||
|
return 0
|
||||||
container = profile.get("CONTAINER_NAME")
|
container = profile.get("CONTAINER_NAME")
|
||||||
if not container:
|
if not container:
|
||||||
raise SystemExit("docker runtime requires CONTAINER_NAME")
|
raise SystemExit("docker runtime requires CONTAINER_NAME")
|
||||||
|
if dry_run:
|
||||||
|
print(f"[dry-run] docker rm -f {container}")
|
||||||
|
return 0
|
||||||
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
||||||
print(f"container {container} removed")
|
print(f"container {container} removed")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def stop(profile: dict[str, str], root: Path) -> int:
|
||||||
|
validate_runtime(profile)
|
||||||
|
if profile["RUNTIME"] == "docker":
|
||||||
|
return _docker_stop(profile)
|
||||||
logs = _log_dir(root, None)
|
logs = _log_dir(root, None)
|
||||||
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
||||||
if pid_file.exists():
|
if pid_file.exists():
|
||||||
@ -113,9 +319,37 @@ def stop(profile: dict[str, str], root: Path) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def status(profile: dict[str, str], root: Path) -> int:
|
def _docker_status(profile: dict[str, str]) -> int:
|
||||||
validate_runtime(profile)
|
node_rank = profile.get("NODE_RANK")
|
||||||
if profile["RUNTIME"] == "docker":
|
if _is_multinode(profile, node_rank):
|
||||||
|
all_ok = 0
|
||||||
|
for rank, host in enumerate(_node_hosts(profile)):
|
||||||
|
node_profile = _sub_node_rank(profile, rank)
|
||||||
|
name = node_profile["CONTAINER_NAME"]
|
||||||
|
target = _node_target(profile, rank)
|
||||||
|
if target:
|
||||||
|
result = subprocess.run(
|
||||||
|
["ssh", "-o", "StrictHostKeyChecking=no", target, f"docker inspect -f '{{{{.State.Status}}}}' {shlex.quote(name)}"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
state = result.stdout.strip() or "missing"
|
||||||
|
print(f"{host}: {name} -> {state}")
|
||||||
|
if result.returncode != 0 or state != "running":
|
||||||
|
all_ok = 1
|
||||||
|
else:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "inspect", "-f", "{{.State.Status}}", name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
state = result.stdout.strip() or "missing"
|
||||||
|
print(f"{host}: {name} -> {state}")
|
||||||
|
if result.returncode != 0 or state != "running":
|
||||||
|
all_ok = 1
|
||||||
|
return all_ok
|
||||||
container = profile.get("CONTAINER_NAME", "")
|
container = profile.get("CONTAINER_NAME", "")
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "inspect", "-f", "{{.State.Status}}", container],
|
["docker", "inspect", "-f", "{{.State.Status}}", container],
|
||||||
@ -125,6 +359,12 @@ def status(profile: dict[str, str], root: Path) -> int:
|
|||||||
)
|
)
|
||||||
print(result.stdout.strip() or "missing")
|
print(result.stdout.strip() or "missing")
|
||||||
return 0 if result.returncode == 0 else 1
|
return 0 if result.returncode == 0 else 1
|
||||||
|
|
||||||
|
|
||||||
|
def status(profile: dict[str, str], root: Path) -> int:
|
||||||
|
validate_runtime(profile)
|
||||||
|
if profile["RUNTIME"] == "docker":
|
||||||
|
return _docker_status(profile)
|
||||||
logs = _log_dir(root, None)
|
logs = _log_dir(root, None)
|
||||||
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
pid_file = logs / f"{profile.get('MODEL_NAME', 'server')}.pid"
|
||||||
if not pid_file.exists():
|
if not pid_file.exists():
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user