diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env index 5f41f2a..681a2dd 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env @@ -47,9 +47,12 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" # Parallel configurations to test. Format: "TP DP" -# Each Ascend910 card has 2 dies; TP addresses dies. TP=8 uses 8 dies (4 cards), -# TP=16 uses all 16 dies (8 cards). Only these two configs are tested. -# Override via PARALLEL_CONFIGS_STR="8,1 16,1" (space-separated). +# A3 910C has 16 dies (8 cards x 2 dies/card). TP/DP address dies, so TP*DP=16 +# uses all dies. The mapping to H20 (8 cards) for fair comparison is: +# A3 TP=4 DP=4 -> 4 dies/replica x 4 replicas == H20 TP=2 DP=4 (2 cards/replica) +# A3 TP=8 DP=2 -> 8 dies/replica x 2 replicas == H20 TP=4 DP=2 (4 cards/replica) +# A3 TP=16 DP=1 -> 16 dies/replica x 1 replica == H20 TP=8 DP=1 (8 cards/replica) +# Override via PARALLEL_CONFIGS_STR="4,4 8,2 16,1" (space-separated). if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then declare -a PARALLEL_CONFIGS=() for pair in $PARALLEL_CONFIGS_STR; do @@ -57,7 +60,8 @@ if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then done else declare -a PARALLEL_CONFIGS=( - "8 1" + "4 4" + "8 2" "16 1" ) fi @@ -74,6 +78,22 @@ BLOCK_SIZE="${BLOCK_SIZE:-128}" MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +# Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). +# GLM-5.2-w4a8c8 ~391 GiB total; with expert-parallel expert weights are sharded +# across TP dies but dense (attn) weights are replicated. Each die has 64GB HBM. +# TP=4: experts/4 + full dense; very tight KV cache -> cap context heavily +# TP=8: experts/8 + full dense; tight KV cache (verified: max_len=16384 fits) +# TP=16: experts/16 + full dense; ample KV cache (full 128K context) +TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.97}" +TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-4096}" +TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-32}" +TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.95}" +TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-16384}" +TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-64}" +TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.92}" +TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" +TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" + # vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. # attention backend for 910C: use the fused/atb attention path. Adjust per image. VLLM_ASCEND_ATTENTION_BACKEND="${VLLM_ASCEND_ATTENTION_BACKEND:-atb}" diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index dcdaf22..bc0cc76 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -80,6 +80,11 @@ engine_build_server_args() { local max_len="$MAX_MODEL_LEN" local max_seqs="$MAX_NUM_SEQS" case "$tp" in + 4) + mem_util="${TP4_GPU_MEMORY_UTILIZATION:-0.97}" + max_len="${TP4_MAX_MODEL_LEN:-4096}" + max_seqs="${TP4_MAX_NUM_SEQS:-32}" + ;; 8) mem_util="${TP8_GPU_MEMORY_UTILIZATION:-0.95}" max_len="${TP8_MAX_MODEL_LEN:-16384}" diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 40ff488..0bcad29 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -34,6 +34,11 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true # Per-TP overrides: TP=8 has tight KV cache on 64GB/die (cap context/seqs), # TP=16 uses all 16 dies and can serve the full 128K context. case "$TP" in + 4) + GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.97}" + MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-4096}" + MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-32}" + ;; 8) GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.95}" MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-16384}"