diff --git a/README.md b/README.md index f6e3977..fd130b0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新(2026-07-20,`903084c`)** +> - H20 / RTX 6000D 的 TPxDP matrix baseline 不再显式限制模型上下文或服务内并发:vLLM 使用默认 `max-model-len` / `max-num-seqs`,SGLang 使用默认 `context-length` / `max-running-requests`。 +> - SGLang 固定使用 `--moe-runner-backend marlin`;vLLM 保持框架默认 MoE backend。 +> - `run_adaptive_concurrency_add16.sh` 在首点 OOM 时会重启服务并按 `C=16 -> 8 -> 1` 回退;仅 H20 / RTX 6000D matrix 默认开启,H200 / P800 保持原行为。 +> - RTX 6000D 旧 SGLang 结果使用 262K context cap,超过该范围或 C>32 的点不可与新 baseline 混用;请使用新的 `RUN_ID` 重跑受影响场景。 + > **项目目的**:当新显卡(GPU/NPU)到货时,用最短时间完成大模型在该平台上的推理性能评估与部署配置选型。 > 当前模型:DeepSeek-V4-Flash(FP8 / INT8);后续接入 GLM5.2,**完全复用**本项目的实验与报告流程。 > 新平台接入 SOP:[`docs/NEW_PLATFORM_GUIDE.md`](docs/NEW_PLATFORM_GUIDE.md)。 diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env index e35c41f..d725207 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env @@ -7,6 +7,9 @@ SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env index 764ffb6..ecda6f8 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env @@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=( ) # SGLang server settings -CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}" +# Context length and scheduler capacity use SGLang's model/default settings. MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 7972a9e..028f0ad 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -72,8 +72,6 @@ engine_build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index c98b531..efbf850 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -72,8 +72,6 @@ engine_build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh index 3fbb7b3..d3020e3 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh @@ -94,8 +94,6 @@ build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$SGLANG_PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh index 69a08b0..cef6ecc 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -27,8 +27,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh index 5753e46..4fce532 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -33,8 +33,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh index 230bf88..3fc17ea 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh @@ -38,8 +38,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$SGLANG_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env index e35c41f..d725207 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -7,6 +7,9 @@ SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env index b9ffb7c..9ab959c 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env @@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=( ) # vLLM server settings -MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +# Context length and scheduler capacity use vLLM's model/default settings. GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-256}" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index f76a93d..ad1801b 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -66,8 +66,6 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index fb18812..a652cff 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -66,8 +66,6 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh index 1db82ea..aef9981 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh @@ -86,8 +86,6 @@ build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh index bab8154..e57c14f 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -34,8 +34,6 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh index 741ca9c..e39bd11 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -39,8 +39,6 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --no-enable-flashinfer-autotune --host 0.0.0.0 --port "$VLLM_PORT" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env index 18854ed..8648353 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env @@ -7,6 +7,9 @@ SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env index 87e9422..7d3ba85 100644 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env @@ -34,8 +34,7 @@ declare -a PARALLEL_CONFIGS=( ) # SGLang server settings (tuned for RTX 6000D 83GB) -CONTEXT_LENGTH="${CONTEXT_LENGTH:-262144}" -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-32}" +# Context length and scheduler capacity use SGLang's model/default settings. MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}" MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index cca4fda..4be536a 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -72,8 +72,6 @@ engine_build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 771b67d..1be9866 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -72,8 +72,6 @@ engine_build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$ENGINE_PORT" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh index ff5131f..27a3c24 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh @@ -94,8 +94,6 @@ build_server_args() { --trust-remote-code --tp-size "$tp" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$SGLANG_PORT" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh index 5346b6b..d59d1aa 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh @@ -27,8 +27,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$PORT" diff --git a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh index 29f27fe..fe758aa 100755 --- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh @@ -33,8 +33,6 @@ SERVER_ARGS=( --trust-remote-code --tp-size "$TP" --moe-runner-backend "$MOE_RUNNER_BACKEND" - --context-length "$CONTEXT_LENGTH" - --max-running-requests "$MAX_RUNNING_REQUESTS" --mem-fraction-static "$MEM_FRACTION_STATIC" --host 0.0.0.0 --port "$PORT" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env index 18854ed..8648353 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env @@ -7,6 +7,9 @@ SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env index 58650eb..5540bb5 100644 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env @@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=( ) # vLLM server settings -MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}" -MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" +# Context length and scheduler capacity use vLLM's model/default settings. GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-256}" diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index ebe4b5e..e09e26c 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -66,8 +66,6 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 854e994..ca24f5f 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -66,8 +66,6 @@ engine_build_server_args() { --block-size "$BLOCK_SIZE" --tensor-parallel-size "$tp" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$ENGINE_PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh index cd07631..8cc539b 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -34,8 +34,6 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$PORT" ) diff --git a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh index d652590..7c28c6f 100755 --- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh +++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -39,8 +39,6 @@ SERVER_ARGS=( --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --max-model-len "$MAX_MODEL_LEN" - --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$VLLM_PORT" ) diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index 12fd1b6..e1e85ac 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -392,6 +392,7 @@ adaptive_run_shape() { local meaningful_gain=1 local point_rc=0 local initial_backoff_active=0 + local initial_backoff_reason="" ADAPTIVE_RESTART_NEEDED=0 ADAPTIVE_FATAL_WORKLOAD=0 @@ -455,6 +456,7 @@ adaptive_run_shape() { if [[ -n "$next_backoff" ]]; then log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}" initial_backoff_active=1 + initial_backoff_reason="TTFT" concurrency="$next_backoff" previous_tps="" plateau_streak=0 @@ -484,9 +486,15 @@ adaptive_run_shape() { log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}" if (( initial_backoff_active == 1 )); then - stop_reason="TTFT_SLO_BACKOFF_FOUND" - shape_status="TTFT_BOUNDARY" - log "initial TTFT backoff found acceptable concurrency c=${concurrency}; stopping shape without probing lower values" + if [[ "$initial_backoff_reason" == "OOM" ]]; then + stop_reason="OOM_BACKOFF_FOUND" + shape_status="OOM_BOUNDARY" + log "initial OOM backoff found acceptable concurrency c=${concurrency}; stopping shape without probing higher values" + else + stop_reason="TTFT_SLO_BACKOFF_FOUND" + shape_status="TTFT_BOUNDARY" + log "initial TTFT backoff found acceptable concurrency c=${concurrency}; stopping shape without probing lower values" + fi break fi if (( plateau_streak >= PLATEAU_PATIENCE )); then @@ -499,6 +507,33 @@ adaptive_run_shape() { adaptive_append_failed_point \ "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + local next_backoff="" + if [[ "${ENABLE_INITIAL_OOM_BACKOFF:-0}" == "1" ]] && \ + (( concurrency == SEARCH_START_CONCURRENCY || initial_backoff_active == 1 )); then + local backoff + for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do + if (( backoff < concurrency )); then + next_backoff="$backoff" + break + fi + done + fi + if [[ -n "$next_backoff" ]]; then + log "probe result c=${concurrency} OOM; restarting server and retrying initial boundary at c=${next_backoff}" + if ! adaptive_restart_server "$tp" "$dp"; then + shape_status="FAILED" + stop_reason="OOM_RESTART_FAILED" + ADAPTIVE_RESTART_NEEDED=1 + break + fi + initial_backoff_active=1 + initial_backoff_reason="OOM" + concurrency="$next_backoff" + previous_tps="" + plateau_streak=0 + plateau_start_concurrency="" + continue + fi shape_status="OOM_BOUNDARY" stop_reason="OOM" ADAPTIVE_RESTART_NEEDED=1