diff --git a/README.md b/README.md index 56eb904..9db2ea3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新(2026-07-21 11:39:51 +0800)** +> - 全量 TP×DP matrix 统一按《模型部署容量参数统计表》显式传入上下文上限、活跃请求上限和显存比例,避免不同引擎/机器因默认值漂移而产生不可比结果。 +> - H20 DSV4-Flash 固定为 1M context、256 活跃请求;RTX 6000D DSV4-Flash 固定为 128K,vLLM 为 128 活跃请求,SGLang 仅测试 TP4/DP2 与 TP8/DP1、为 64 活跃请求;P800 DSV4-Flash 固定为 64K context 且跳过不可部署的 TP2/DP4;H20 GLM-5.2 仅测试 TP8/DP1、固定为 256K/128。 + > **更新(2026-07-20 17:55:20 +0800)** > - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`,使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph;Docker 与 native 启动入口均已覆盖。 diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env index d725207..3b05c7c 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env @@ -6,7 +6,7 @@ # PLATEAU_PATIENCE consecutive points. SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" -SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-256}" # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env index ec19aff..a3c4e1b 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env @@ -32,10 +32,11 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) -# SGLang server settings -# Context length and scheduler capacity use SGLang's model/default settings. -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" +# SGLang server settings, verified for H20 / DeepSeek-V4-Flash. +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" +CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}" # Keep high-concurrency decode on captured CUDA Graphs on H20. CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-128}" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 028f0ad..f0c1e77 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -73,6 +73,8 @@ engine_build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index efbf850..3620ef2 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -73,6 +73,8 @@ engine_build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh index d3020e3..4664811 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh @@ -95,6 +95,8 @@ build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$SGLANG_PORT" ) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh index cef6ecc..c2666ab 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -28,6 +28,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$PORT" ) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh index 3b65d8c..3747750 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -34,6 +34,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$PORT" ) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh index 1425f48..d4c4ced 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -39,6 +39,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$SGLANG_PORT" ) diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env index d725207..3b05c7c 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -6,7 +6,7 @@ # PLATEAU_PATIENCE consecutive points. SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" -SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-256}" # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env index 571404a..744216a 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env @@ -32,11 +32,12 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) -# vLLM server settings -# Context length and scheduler capacity use vLLM's model/default settings. +# vLLM server settings, verified for H20 / DeepSeek-V4-Flash. GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" # Deployment switch. 0 = native vllm venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index ad1801b..f76a93d 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -66,6 +66,8 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index a652cff..fb18812 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -66,6 +66,8 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh index aef9981..1db82ea 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh @@ -86,6 +86,8 @@ build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh index e57c14f..bab8154 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -34,6 +34,8 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh index e39bd11..741ca9c 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -39,6 +39,8 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_config.env index 5c8d5ea..3721b18 100644 --- a/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -6,7 +6,7 @@ # PLATEAU_PATIENCE consecutive points. SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" -SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}" # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" diff --git a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index ad1801b..f76a93d 100755 --- a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -66,6 +66,8 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index c30cbcf..fb2229d 100755 --- a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -66,6 +66,7 @@ engine_build_server_args() { --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh index aef9981..1db82ea 100755 --- a/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh @@ -86,6 +86,8 @@ build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh index e39bd11..741ca9c 100755 --- a/experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -39,6 +39,8 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env index 5a3d8a3..b3fd425 100644 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env @@ -27,12 +27,8 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" # Parallel configurations to test. Format: "TP DP" -# Note: TP=2/DP=4 fails at startup with a weight-loading OOM. Root cause: -# expert weights are sharded only across the TP group (no --ep-size is -# passed, so experts fall back to TP sharding), and DP replicas do not -# share expert shards. The INT8 model holds ~264 GiB of routed-expert -# weights, so TP=2 needs ~137 GiB per XPU (>96 GiB). TP=2/DP=4 -# is kept in the default list so the matrix records the failure and moves on. +# TP=2/DP=4 is excluded because routed-expert weight loading OOMs before +# capacity initialization; DP replicas do not share TP-sharded expert weights. # Override with a space-separated list of comma pairs, e.g. # PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1. if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then @@ -42,15 +38,13 @@ if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then done else declare -a PARALLEL_CONFIGS=( - "2 4" "4 2" "8 1" ) fi -# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so -# the context length is capped well below the H20 1M setting. -CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}" +# SGLang server settings, verified for P800 / DeepSeek-V4-Flash-INT8. +CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env index 2342169..a7408ed 100644 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env @@ -4,7 +4,6 @@ # --ep-size (expert parallel; probes whether EP makes TP2/DP4 viable) # EAGLE speculative decoding (steps=3, topk=1, draft=4) # --chunked-prefill-size 8192 --max-prefill-tokens 16384 -# --max-running-requests 64 # Tests SGLang with three parallel configurations: # TP=2, DP=4 -> 2 XPUs per replica, 4 replicas # TP=4, DP=2 -> 4 XPUs per replica, 2 replicas @@ -52,9 +51,8 @@ else ) fi -# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so -# the context length is capped well below the H20 1M setting. -CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}" +# SGLang server settings, verified for P800 / DeepSeek-V4-Flash-INT8. +CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}" MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" # Official w8a8 parameters (from /workspace/ds_v4/w8a8/run_server.sh). @@ -63,7 +61,6 @@ MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" EP_SIZE="${EP_SIZE:-}" CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-8192}" MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-16384}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" # EAGLE speculative decoding (official defaults: steps=3, topk=1, draft=4). SPEC_NUM_STEPS="${SPEC_NUM_STEPS:-3}" SPEC_EAGLE_TOPK="${SPEC_EAGLE_TOPK:-1}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh index 4200124..8b6d2fa 100755 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh @@ -72,7 +72,7 @@ engine_build_server_args() { if (( dp == 1 )); then ep_args="--ep-size ${EP_SIZE:-$tp}" fi - local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" if (( dp > 1 )); then args="${args} --dp-size ${dp}" fi diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh index 5118d31..5824e57 100755 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh @@ -72,7 +72,7 @@ engine_build_server_args() { if (( dp == 1 )); then ep_args="--ep-size ${EP_SIZE:-$tp}" fi - local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" if (( dp > 1 )); then args="${args} --dp-size ${dp}" fi diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh index c78c399..31888db 100755 --- a/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh @@ -93,14 +93,13 @@ env_args=( # experiment) PLUS the official w8a8 run_server.sh parameters: # EAGLE speculative decoding (steps/topk/draft from config.env) # --chunked-prefill-size / --max-prefill-tokens (official long-prefill chunking) -# --max-running-requests 64 (official cap) # --ep-size is only added when DP=1: this vendor image crashes with # ZeroDivisionError when EP spans DP replicas (probed 2026-07-17). ep_args="" if [[ "$DP" -eq 1 ]]; then ep_args="--ep-size ${EP_SIZE:-$TP}" fi -launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" if [[ "$DP" -gt 1 ]]; then launch_args="${launch_args} --dp-size ${DP}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env index 8648353..3cf7dd7 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -6,7 +6,7 @@ # PLATEAU_PATIENCE consecutive points. SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" -SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}" # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env index 58bef1d..825b3ea 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -1,7 +1,8 @@ #!/usr/bin/env bash # TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using SGLang. -# Tests SGLang with three parallel configurations: -# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache +# initialization on this machine. +# Tests SGLang with two parallel configurations: # TP=4, DP=2 -> 4 GPUs per replica, 2 replicas # TP=8, DP=1 -> 8 GPUs, no data parallelism @@ -28,19 +29,15 @@ RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" # Parallel configurations to test. Format: "TP DP" declare -a PARALLEL_CONFIGS=( - "2 4" "4 2" "8 1" ) -# SGLang server settings (tuned for RTX 6000D 83GB) -# Context length and scheduler capacity use SGLang's model/default settings. -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}" -MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" - -# RTX 6000D cannot capture DSV4 decode graphs at the default large batches. -# This limits graph pre-capture only; it does not cap request concurrency. -CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-16}" +# SGLang server settings, verified for RTX 6000D / DeepSeek-V4-Flash. +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}" +MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}" +CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" # Deployment switch. 0 = native sglang venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 286d9b9..39ed951 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -73,15 +73,14 @@ engine_build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$ENGINE_PORT" ) if (( dp > 1 )); then args+=(--dp-size "$dp") fi - if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then - args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") - fi printf '%q ' "${args[@]}" } diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index ec77610..ab26fe1 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -73,15 +73,14 @@ engine_build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$ENGINE_PORT" ) if (( dp > 1 )); then args+=(--dp-size "$dp") fi - if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then - args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") - fi printf '%q ' "${args[@]}" } diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh index 27a3c24..5b3a6d3 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh @@ -95,6 +95,8 @@ build_server_args() { --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$SGLANG_PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh index d59d1aa..cc8713a 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -28,6 +28,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh index 23d0901..e251dd5 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -34,6 +34,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$PORT" ) @@ -44,10 +46,6 @@ if [[ "$DP" -gt 1 ]]; then ) fi -if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then - SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") -fi - SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ===" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh index ccbd125..b7e69e3 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -40,6 +40,8 @@ SERVER_ARGS=( --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" --mem-fraction-static "$MEM_FRACTION_STATIC" + --context-length "$CONTEXT_LENGTH" + --max-running-requests "$MAX_RUNNING_REQUESTS" --host 0.0.0.0 --port "$SGLANG_PORT" ) @@ -50,10 +52,6 @@ if [[ "$DP" -gt 1 ]]; then ) fi -if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then - SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE") -fi - SERVER_ARGS_STR="${SERVER_ARGS[*]}" echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ===" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env index 8648353..cdf35c2 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env @@ -6,7 +6,7 @@ # PLATEAU_PATIENCE consecutive points. SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" -SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}" # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env index 4585543..3df6501 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env @@ -32,11 +32,12 @@ declare -a PARALLEL_CONFIGS=( "8 1" ) -# vLLM server settings -# Context length and scheduler capacity use vLLM's model/default settings. +# vLLM server settings, verified for RTX 6000D / DeepSeek-V4-Flash. GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" # Deployment switch. 0 = native vllm venv, 1 = Docker. USE_DOCKER="${USE_DOCKER:-1}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index e09e26c..ebe4b5e 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -66,6 +66,8 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index ca24f5f..854e994 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -66,6 +66,8 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh index 8cc539b..cd07631 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -34,6 +34,8 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh index 7c28c6f..d652590 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -39,6 +39,8 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$VLLM_PORT" )