From 99a22f05b82226bd402bc708a316bfda41e1d61c Mon Sep 17 00:00:00 2001 From: shishi Date: Wed, 29 Jul 2026 15:09:27 +0800 Subject: [PATCH] fix(dsv4): align launch params with official A3 tutorial (fixes DP die allocation) The previous params caused all 4 DP replicas to bind to the same 4 dies (die 0-3), leaving 12 dies idle. Root cause was a combination of missing official params + extra non-official params that interfered with DP worker device placement. Verified: with aligned params, TP=4 DP=4 correctly distributes 16 workers across all 16 dies (8 cards x 2 dies), each at ~57 GB HBM (91% util). Changes (align to docs.vllm.ai A3 tutorial): - Add --max-num-batched-tokens 10240 (was missing; affects DP scheduling) - Add --api-server-count 1 (was missing; without it vllm spawns N API servers for N DP ranks, disturbing device assignment) - Remove --kv-cache-dtype fp8 (official uses default bfloat16) - Remove --trust-remote-code (official doesn't use it for DSV4) - Remove enable_dsa_cp from additional-config (official doesn't have it) - max-model-len: per-TP caps (32768/65536/131072) -> 1048576 for all TPs (official uses full 1M; the caps were over-cautious) - max-num-seqs: per-TP (128/256/256) -> 64 for all (official value) --- .../dsv4_910c_vllm_tp_dp_matrix/config.env | 24 +++++++++++-------- .../run_adaptive_concurrency_add16.sh | 6 ++--- .../start_vllm_docker.sh | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index e85a12d..83290d5 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -79,10 +79,14 @@ fi # matrix sweep (extend to 1M via matrix.json once TP=8 is verified). # - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9). GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" -KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}" BLOCK_SIZE="${BLOCK_SIZE:-128}" -MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-64}" +# Max tokens per batch (official A3 tutorial: 10240). +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-10240}" +# API server count (official: 1). +API_SERVER_COUNT="${API_SERVER_COUNT:-1}" # Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). # DSV4-Flash w8a8 weight per die = ~280GiB / TP. Each die has 64GB HBM. @@ -90,14 +94,14 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" # TP=8: ~35 GiB/die weights -> 29 GiB for KV cache; balanced # TP=16: ~17.5 GiB/die weights -> 46 GiB for KV cache; full context TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" -TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-32768}" -TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-128}" +TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-1048576}" +TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-64}" TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" -TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-65536}" -TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-256}" +TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-1048576}" +TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-64}" TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" -TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" -TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" +TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-1048576}" +TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-64}" # vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. # attention backend for 910C: use the fused/atb attention path. Adjust per image. @@ -123,7 +127,7 @@ DSV4_SAFETENSORS_LOAD_STRATEGY="${DSV4_SAFETENSORS_LOAD_STRATEGY:-prefetch}" # in the launcher to avoid shell mangling. DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}" DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}" -DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"enable_dsa_cp\": true,\"multistream_overlap_shared_expert\":true}}" +DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"multistream_overlap_shared_expert\":true}}" DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}" DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index be790d1..ddf1bd0 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -99,13 +99,13 @@ engine_build_server_args() { local -a args=( vllm serve "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" - --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" - --block-size "$BLOCK_SIZE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$mem_util" --max-model-len "$max_len" --max-num-seqs "$max_seqs" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 1e58657..33e8afd 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -59,13 +59,13 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" - --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --host 0.0.0.0 --port "$PORT" )