fix(dsv4): align launch params with official A3 tutorial (fixes DP die allocation)

The previous params caused all 4 DP replicas to bind to the same 4 dies
(die 0-3), leaving 12 dies idle. Root cause was a combination of missing
official params + extra non-official params that interfered with DP
worker device placement.

Verified: with aligned params, TP=4 DP=4 correctly distributes 16 workers
across all 16 dies (8 cards x 2 dies), each at ~57 GB HBM (91% util).

Changes (align to docs.vllm.ai A3 tutorial):
- Add --max-num-batched-tokens 10240 (was missing; affects DP scheduling)
- Add --api-server-count 1 (was missing; without it vllm spawns N API
  servers for N DP ranks, disturbing device assignment)
- Remove --kv-cache-dtype fp8 (official uses default bfloat16)
- Remove --trust-remote-code (official doesn't use it for DSV4)
- Remove enable_dsa_cp from additional-config (official doesn't have it)
- max-model-len: per-TP caps (32768/65536/131072) -> 1048576 for all TPs
  (official uses full 1M; the caps were over-cautious)
- max-num-seqs: per-TP (128/256/256) -> 64 for all (official value)
This commit is contained in:
shishi 2026-07-29 15:09:27 +08:00
parent c222ed98b2
commit 99a22f05b8
3 changed files with 19 additions and 15 deletions

View File

@ -79,10 +79,14 @@ fi
# matrix sweep (extend to 1M via matrix.json once TP=8 is verified). # matrix sweep (extend to 1M via matrix.json once TP=8 is verified).
# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9). # - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9).
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}"
BLOCK_SIZE="${BLOCK_SIZE:-128}" BLOCK_SIZE="${BLOCK_SIZE:-128}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-64}"
# Max tokens per batch (official A3 tutorial: 10240).
MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-10240}"
# API server count (official: 1).
API_SERVER_COUNT="${API_SERVER_COUNT:-1}"
# Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). # Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP).
# DSV4-Flash w8a8 weight per die = ~280GiB / TP. Each die has 64GB HBM. # DSV4-Flash w8a8 weight per die = ~280GiB / TP. Each die has 64GB HBM.
@ -90,14 +94,14 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
# TP=8: ~35 GiB/die weights -> 29 GiB for KV cache; balanced # TP=8: ~35 GiB/die weights -> 29 GiB for KV cache; balanced
# TP=16: ~17.5 GiB/die weights -> 46 GiB for KV cache; full context # TP=16: ~17.5 GiB/die weights -> 46 GiB for KV cache; full context
TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}"
TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-32768}" TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-1048576}"
TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-128}" TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-64}"
TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}"
TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-65536}" TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-1048576}"
TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-256}" TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-64}"
TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}"
TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-1048576}"
TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-64}"
# vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. # vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh.
# attention backend for 910C: use the fused/atb attention path. Adjust per image. # attention backend for 910C: use the fused/atb attention path. Adjust per image.
@ -123,7 +127,7 @@ DSV4_SAFETENSORS_LOAD_STRATEGY="${DSV4_SAFETENSORS_LOAD_STRATEGY:-prefetch}"
# in the launcher to avoid shell mangling. # in the launcher to avoid shell mangling.
DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}" DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}"
DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}" DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}"
DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"enable_dsa_cp\": true,\"multistream_overlap_shared_expert\":true}}" DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"multistream_overlap_shared_expert\":true}}"
DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}" DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}"
DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}"

View File

@ -99,13 +99,13 @@ engine_build_server_args() {
local -a args=( local -a args=(
vllm serve "$MODEL_PATH" vllm serve "$MODEL_PATH"
--served-model-name "$SERVED_MODEL_NAME" --served-model-name "$SERVED_MODEL_NAME"
--trust-remote-code --block-size "$BLOCK_SIZE"
--kv-cache-dtype "$KV_CACHE_DTYPE"
--block-size "$BLOCK_SIZE"
--tensor-parallel-size "$tp" --tensor-parallel-size "$tp"
--gpu-memory-utilization "$mem_util" --gpu-memory-utilization "$mem_util"
--max-model-len "$max_len" --max-model-len "$max_len"
--max-num-seqs "$max_seqs" --max-num-seqs "$max_seqs"
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
--api-server-count "$API_SERVER_COUNT"
--host 0.0.0.0 --host 0.0.0.0
--port "$ENGINE_PORT" --port "$ENGINE_PORT"
) )

View File

@ -59,13 +59,13 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true
SERVER_ARGS=( SERVER_ARGS=(
"$MODEL_PATH" "$MODEL_PATH"
--served-model-name "$SERVED_MODEL_NAME" --served-model-name "$SERVED_MODEL_NAME"
--trust-remote-code
--kv-cache-dtype "$KV_CACHE_DTYPE"
--block-size "$BLOCK_SIZE" --block-size "$BLOCK_SIZE"
--tensor-parallel-size "$TP" --tensor-parallel-size "$TP"
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
--max-model-len "$MAX_MODEL_LEN" --max-model-len "$MAX_MODEL_LEN"
--max-num-seqs "$MAX_NUM_SEQS" --max-num-seqs "$MAX_NUM_SEQS"
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
--api-server-count "$API_SERVER_COUNT"
--host 0.0.0.0 --host 0.0.0.0
--port "$PORT" --port "$PORT"
) )