diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index 9c92a74..e85a12d 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -49,9 +49,12 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" # Parallel configurations to test. Format: "TP DP" -# DSV4-Flash w8a8 routed-expert weights ~280 GiB total. With expert-parallel the -# weight load is sharded across TP dies, but w8a8 still leaves a heavy per-die -# footprint. TP*DP must equal 16 (8 cards x 2 dies). +# A3 910C has 16 dies (8 cards x 2 dies/card). TP/DP address dies, so TP*DP=16 +# uses all dies. DSV4-Flash w8a8 weights ~280 GiB total, sharded across TP dies: +# TP=4 DP=4 -> 4 dies/replica x 4 replicas (39 GiB/die, tight KV cache) +# TP=8 DP=2 -> 8 dies/replica x 2 replicas (35 GiB/die, balanced) +# TP=16 DP=1 -> 16 dies/replica x 1 replica (17.5 GiB/die, max KV cache) +# Note: A3 TP=4 == H20 TP=4 per-card-equivalent (A3 has 2 dies/card). # TP=2, DP=4 -> 4 dies/replica x 4 replicas (smallest TP, most replicas) # TP=4, DP=2 -> 4 dies/replica x 2 replicas # TP=8, DP=1 -> 8 dies/replica x 1 replica (largest TP, max weight sharding) @@ -62,9 +65,9 @@ if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then done else declare -a PARALLEL_CONFIGS=( - "2 4" - "4 2" - "8 1" + "4 4" + "8 2" + "16 1" ) fi @@ -81,6 +84,21 @@ BLOCK_SIZE="${BLOCK_SIZE:-128}" MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +# Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). +# DSV4-Flash w8a8 weight per die = ~280GiB / TP. Each die has 64GB HBM. +# TP=4: ~39 GiB/die weights -> 25 GiB for KV cache; cap context to avoid OOM +# TP=8: ~35 GiB/die weights -> 29 GiB for KV cache; balanced +# TP=16: ~17.5 GiB/die weights -> 46 GiB for KV cache; full context +TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" +TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-32768}" +TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-128}" +TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" +TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-65536}" +TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-256}" +TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" +TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" +TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" + # vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. # attention backend for 910C: use the fused/atb attention path. Adjust per image. VLLM_ASCEND_ATTENTION_BACKEND="${VLLM_ASCEND_ATTENTION_BACKEND:-atb}" diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 3242291..be790d1 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -74,6 +74,28 @@ engine_stop_server() { engine_build_server_args() { local tp="$1" local dp="$2" + # Per-TP overrides (must match start_vllm_docker.sh case $TP). + local mem_util="$GPU_MEMORY_UTILIZATION" + local max_len="$MAX_MODEL_LEN" + local max_seqs="$MAX_NUM_SEQS" + case "$tp" in + 4) + mem_util="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" + max_len="${TP4_MAX_MODEL_LEN:-32768}" + max_seqs="${TP4_MAX_NUM_SEQS:-128}" + ;; + 8) + mem_util="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" + max_len="${TP8_MAX_MODEL_LEN:-65536}" + max_seqs="${TP8_MAX_NUM_SEQS:-256}" + ;; + 16) + mem_util="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" + max_len="${TP16_MAX_MODEL_LEN:-131072}" + max_seqs="${TP16_MAX_NUM_SEQS:-256}" + ;; + esac + log "server args tp=${tp} dp=${dp} gpu_mem_util=${mem_util} max_model_len=${max_len} max_num_seqs=${max_seqs}" >&2 local -a args=( vllm serve "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" @@ -81,9 +103,9 @@ engine_build_server_args() { --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" + --gpu-memory-utilization "$mem_util" + --max-model-len "$max_len" + --max-num-seqs "$max_seqs" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index f17247d..1e58657 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -18,6 +18,26 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" +# Per-TP parameter overrides (synced from glm52 fix 6c81183). +# DSV4-Flash w8a8 weight per die = ~280GiB / TP; each die has 64GB HBM. +case "$TP" in + 4) + GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" + MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-32768}" + MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-128}" + ;; + 8) + GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" + MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-65536}" + MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-256}" + ;; + 16) + GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" + MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" + MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" + ;; +esac + RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"