fix(910c/glm52): 对齐官方A3教程参数(修DP die分配问题,同dsv4 99a22f0)
dsv4实验发现DP副本绑定到同一组die的问题(99a22f0),glm52存在相同问题: 缺--max-num-batched-tokens和--api-server-count导致DP worker设备分配异常 (不加api-server-count时vllm为N个DP rank启动N个API server)。 对齐docs.vllm.ai GLM5.2 A3官方教程: - 加 --max-num-batched-tokens 8192 (官方值,影响DP调度) - 加 --api-server-count 1 (官方值,避免多API server干扰设备分配) - 去掉 --kv-cache-dtype fp8 (官方不指定,用默认bfloat16;且此镜像fp8本就未生效) - 保留 --trust-remote-code / --enable-expert-parallel / enable_dsa_cp (官方有)
This commit is contained in:
parent
99a22f05b8
commit
455a78161b
@ -73,10 +73,15 @@ fi
|
|||||||
# - MAX_MODEL_LEN: GLM-5.2 supports up to 128K context; cap at 131072.
|
# - MAX_MODEL_LEN: GLM-5.2 supports up to 128K context; cap at 131072.
|
||||||
# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9 mirrors H20).
|
# - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9 mirrors H20).
|
||||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}"
|
||||||
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
|
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}"
|
||||||
BLOCK_SIZE="${BLOCK_SIZE:-128}"
|
BLOCK_SIZE="${BLOCK_SIZE:-128}"
|
||||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
|
||||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
|
MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
|
||||||
|
# Official A3 tutorial params (docs.vllm.ai GLM5.2): max-num-batched-tokens=8192,
|
||||||
|
# api-server-count=1 (without it, vllm spawns N API servers for N DP ranks,
|
||||||
|
# disturbing DP worker device placement -- same issue as dsv4 fix 99a22f0).
|
||||||
|
MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-8192}"
|
||||||
|
API_SERVER_COUNT="${API_SERVER_COUNT:-1}"
|
||||||
|
|
||||||
# Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP).
|
# Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP).
|
||||||
# GLM-5.2-w4a8c8 ~391 GiB total; with expert-parallel expert weights are sharded
|
# GLM-5.2-w4a8c8 ~391 GiB total; with expert-parallel expert weights are sharded
|
||||||
|
|||||||
@ -107,13 +107,14 @@ engine_build_server_args() {
|
|||||||
vllm serve "$MODEL_PATH"
|
vllm serve "$MODEL_PATH"
|
||||||
--served-model-name "$SERVED_MODEL_NAME"
|
--served-model-name "$SERVED_MODEL_NAME"
|
||||||
--trust-remote-code
|
--trust-remote-code
|
||||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
|
||||||
--block-size "$BLOCK_SIZE"
|
--block-size "$BLOCK_SIZE"
|
||||||
--tensor-parallel-size "$tp"
|
--tensor-parallel-size "$tp"
|
||||||
--enable-expert-parallel
|
--enable-expert-parallel
|
||||||
--gpu-memory-utilization "$mem_util"
|
--gpu-memory-utilization "$mem_util"
|
||||||
--max-model-len "$max_len"
|
--max-model-len "$max_len"
|
||||||
--max-num-seqs "$max_seqs"
|
--max-num-seqs "$max_seqs"
|
||||||
|
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
|
||||||
|
--api-server-count "$API_SERVER_COUNT"
|
||||||
--compilation-config "$compilation_config"
|
--compilation-config "$compilation_config"
|
||||||
--additional-config "$additional_config"
|
--additional-config "$additional_config"
|
||||||
--host 0.0.0.0
|
--host 0.0.0.0
|
||||||
|
|||||||
@ -55,13 +55,14 @@ SERVER_ARGS=(
|
|||||||
"$MODEL_PATH"
|
"$MODEL_PATH"
|
||||||
--served-model-name "$SERVED_MODEL_NAME"
|
--served-model-name "$SERVED_MODEL_NAME"
|
||||||
--trust-remote-code
|
--trust-remote-code
|
||||||
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
|
||||||
--block-size "$BLOCK_SIZE"
|
--block-size "$BLOCK_SIZE"
|
||||||
--tensor-parallel-size "$TP"
|
--tensor-parallel-size "$TP"
|
||||||
--enable-expert-parallel
|
--enable-expert-parallel
|
||||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
--max-model-len "$MAX_MODEL_LEN"
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
--max-num-seqs "$MAX_NUM_SEQS"
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS"
|
||||||
|
--api-server-count "$API_SERVER_COUNT"
|
||||||
--host 0.0.0.0
|
--host 0.0.0.0
|
||||||
--port "$PORT"
|
--port "$PORT"
|
||||||
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
--compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user