diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env index e85a12d..83290d5 100644 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/config.env @@ -79,10 +79,14 @@ fi # matrix sweep (extend to 1M via matrix.json once TP=8 is verified). # - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9). GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" -KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-}" BLOCK_SIZE="${BLOCK_SIZE:-128}" -MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-64}" +# Max tokens per batch (official A3 tutorial: 10240). +MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-10240}" +# API server count (official: 1). +API_SERVER_COUNT="${API_SERVER_COUNT:-1}" # Per-TP parameter overrides (applied in start_vllm_docker.sh via case $TP). # DSV4-Flash w8a8 weight per die = ~280GiB / TP. Each die has 64GB HBM. @@ -90,14 +94,14 @@ MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" # TP=8: ~35 GiB/die weights -> 29 GiB for KV cache; balanced # TP=16: ~17.5 GiB/die weights -> 46 GiB for KV cache; full context TP4_GPU_MEMORY_UTILIZATION="${TP4_GPU_MEMORY_UTILIZATION:-0.9}" -TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-32768}" -TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-128}" +TP4_MAX_MODEL_LEN="${TP4_MAX_MODEL_LEN:-1048576}" +TP4_MAX_NUM_SEQS="${TP4_MAX_NUM_SEQS:-64}" TP8_GPU_MEMORY_UTILIZATION="${TP8_GPU_MEMORY_UTILIZATION:-0.9}" -TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-65536}" -TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-256}" +TP8_MAX_MODEL_LEN="${TP8_MAX_MODEL_LEN:-1048576}" +TP8_MAX_NUM_SEQS="${TP8_MAX_NUM_SEQS:-64}" TP16_GPU_MEMORY_UTILIZATION="${TP16_GPU_MEMORY_UTILIZATION:-0.9}" -TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-131072}" -TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-256}" +TP16_MAX_MODEL_LEN="${TP16_MAX_MODEL_LEN:-1048576}" +TP16_MAX_NUM_SEQS="${TP16_MAX_NUM_SEQS:-64}" # vLLM-Ascend-specific launch flags injected by start_vllm_docker.sh. # attention backend for 910C: use the fused/atb attention path. Adjust per image. @@ -123,7 +127,7 @@ DSV4_SAFETENSORS_LOAD_STRATEGY="${DSV4_SAFETENSORS_LOAD_STRATEGY:-prefetch}" # in the launcher to avoid shell mangling. DSV4_SPEC_CONFIG="${DSV4_SPEC_CONFIG:-{\"num_speculative_tokens\": 1, \"method\": \"mtp\", \"enforce_eager\": true}}" DSV4_COMPILATION_CONFIG="${DSV4_COMPILATION_CONFIG:-{\"cudagraph_mode\": \"FULL_DECODE_ONLY\"}}" -DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"enable_dsa_cp\": true,\"multistream_overlap_shared_expert\":true}}" +DSV4_ADDITIONAL_CONFIG="${DSV4_ADDITIONAL_CONFIG:-{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\": true,\"multistream_overlap_shared_expert\":true}}" DSV4_ENABLE_EXPERT_PARALLEL="${DSV4_ENABLE_EXPERT_PARALLEL:-1}" DSV4_ENABLE_ASYNC_SCHEDULING="${DSV4_ENABLE_ASYNC_SCHEDULING:-1}" diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index be790d1..ddf1bd0 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -99,13 +99,13 @@ engine_build_server_args() { local -a args=( vllm serve "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" - --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" - --block-size "$BLOCK_SIZE" + --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$mem_util" --max-model-len "$max_len" --max-num-seqs "$max_seqs" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 1e58657..33e8afd 100755 --- a/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/dsv4_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -59,13 +59,13 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true SERVER_ARGS=( "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" - --trust-remote-code - --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" + --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" + --api-server-count "$API_SERVER_COUNT" --host 0.0.0.0 --port "$PORT" )