diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env index 681a2dd..e2ab628 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env @@ -73,10 +73,15 @@ fi # - MAX_MODEL_LEN: GLM-5.2 supports up to 128K context; cap at 131072. # - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9 mirrors H20). GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}" -KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}" BLOCK_SIZE="${BLOCK_SIZE:-128}" MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +# Official A3 tutorial params (docs.vllm.ai GLM5.2): max-num-batched-tokens=8192, +# api-server-count=1 (without it, vllm spawns N API servers for N DP ranks, +# disturbing DP worker device placement -- same issue as dsv4 fix 99a22f0). +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-8192}" +API_SERVER_COUNT="${API_SERVER_COUNT:-1}" # Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). # GLM-5.2-w4a8c8 ~391 GiB total; with expert-parallel expert weights are sharded diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index bc0cc76..8312b3c 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -107,13 +107,14 @@ engine_build_server_args() { vllm serve "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --enable-expert-parallel --gpu-memory-utilization "$mem_util" --max-model-len "$max_len" --max-num-seqs "$max_seqs" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --compilation-config "$compilation_config" --additional-config "$additional_config" --host 0.0.0.0 diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 0bcad29..4641b38 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -55,13 +55,14 @@ SERVER_ARGS=( "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --enable-expert-parallel --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --host 0.0.0.0 --port "$PORT" --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'