[Feat] apply validated deployment capacity caps

README.md

@@ -1,5 +1,9 @@
 # sskj — 多平台大模型推理性能基准测试项目
 
+> **更新（2026-07-21 11:39:51 +0800）**
+> - 全量 TP×DP matrix 统一按《模型部署容量参数统计表》显式传入上下文上限、活跃请求上限和显存比例，避免不同引擎/机器因默认值漂移而产生不可比结果。
+> - H20 DSV4-Flash 固定为 1M context、256 活跃请求；RTX 6000D DSV4-Flash 固定为 128K，vLLM 为 128 活跃请求，SGLang 仅测试 TP4/DP2 与 TP8/DP1、为 64 活跃请求；P800 DSV4-Flash 固定为 64K context 且跳过不可部署的 TP2/DP4；H20 GLM-5.2 仅测试 TP8/DP1、固定为 256K/128。
+
 > **更新（2026-07-20 17:55:20 +0800）**
 > - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`，使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph；Docker 与 native 启动入口均已覆盖。
 

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env

@@ -6,7 +6,7 @@
 # PLATEAU_PATIENCE consecutive points.
 
 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
-SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-256}"
 
 # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
 ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env

@@ -32,10 +32,11 @@ declare -a PARALLEL_CONFIGS=(
   "8 1"
 )
 
-# SGLang server settings
+# SGLang server settings, verified for H20 / DeepSeek-V4-Flash.
-# Context length and scheduler capacity use SGLang's model/default settings.
+MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
-MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}"
 MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"
+CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}"
+MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}"
 # Keep high-concurrency decode on captured CUDA Graphs on H20.
 CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-128}"
 

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -73,6 +73,8 @@ engine_build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -73,6 +73,8 @@ engine_build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh

@@ -95,6 +95,8 @@ build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$SGLANG_PORT"
   )

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh

@@ -28,6 +28,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$PORT"
 )

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh

@@ -34,6 +34,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$PORT"
 )

experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh

@@ -39,6 +39,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$SGLANG_PORT"
 )

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env

@@ -6,7 +6,7 @@
 # PLATEAU_PATIENCE consecutive points.
 
 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
-SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-256}"
 
 # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
 ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env

@@ -32,11 +32,12 @@ declare -a PARALLEL_CONFIGS=(
   "8 1"
 )
 
-# vLLM server settings
+# vLLM server settings, verified for H20 / DeepSeek-V4-Flash.
-# Context length and scheduler capacity use vLLM's model/default settings.
 GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
 KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
 BLOCK_SIZE="${BLOCK_SIZE:-256}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
 
 # Deployment switch. 0 = native vllm venv, 1 = Docker.
 USE_DOCKER="${USE_DOCKER:-1}"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -66,6 +66,8 @@ engine_build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$ENGINE_PORT"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -66,6 +66,8 @@ engine_build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$ENGINE_PORT"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh

@@ -86,6 +86,8 @@ build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$VLLM_PORT"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh

@@ -34,6 +34,8 @@ SERVER_ARGS=(
   --block-size "$BLOCK_SIZE"
   --tensor-parallel-size "$TP"
   --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
   --no-enable-flashinfer-autotune
   --host 0.0.0.0
   --port "$PORT"

experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh

@@ -39,6 +39,8 @@ SERVER_ARGS=(
   --block-size "$BLOCK_SIZE"
   --tensor-parallel-size "$TP"
   --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
   --no-enable-flashinfer-autotune
   --host 0.0.0.0
   --port "$VLLM_PORT"

experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_config.env

@@ -6,7 +6,7 @@
 # PLATEAU_PATIENCE consecutive points.
 
 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
-SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}"
 
 # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
 ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"

experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -66,6 +66,8 @@ engine_build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$ENGINE_PORT"

experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -66,6 +66,7 @@ engine_build_server_args() {
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
     --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$ENGINE_PORT"

experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh

@@ -86,6 +86,8 @@ build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --no-enable-flashinfer-autotune
     --host 0.0.0.0
     --port "$VLLM_PORT"

experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh

@@ -39,6 +39,8 @@ SERVER_ARGS=(
   --block-size "$BLOCK_SIZE"
   --tensor-parallel-size "$TP"
   --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
   --no-enable-flashinfer-autotune
   --host 0.0.0.0
   --port "$VLLM_PORT"

experiments/p800/dsv4_p800_sglang_tp_dp_matrix/config.env

@@ -27,12 +27,8 @@ export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
 RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
 
 # Parallel configurations to test. Format: "TP DP"
-# Note: TP=2/DP=4 fails at startup with a weight-loading OOM. Root cause:
+# TP=2/DP=4 is excluded because routed-expert weight loading OOMs before
-# expert weights are sharded only across the TP group (no --ep-size is
+# capacity initialization; DP replicas do not share TP-sharded expert weights.
-# passed, so experts fall back to TP sharding), and DP replicas do not
-# share expert shards. The INT8 model holds ~264 GiB of routed-expert
-# weights, so TP=2 needs ~137 GiB per XPU (>96 GiB). TP=2/DP=4
-# is kept in the default list so the matrix records the failure and moves on.
 # Override with a space-separated list of comma pairs, e.g.
 # PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1.
 if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
@@ -42,15 +38,13 @@ if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
   done
 else
   declare -a PARALLEL_CONFIGS=(
-    "2 4"
     "4 2"
     "8 1"
   )
 fi
 
-# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so
+# SGLang server settings, verified for P800 / DeepSeek-V4-Flash-INT8.
-# the context length is capped well below the H20 1M setting.
+CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}"
-CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}"
 MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
 
 DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"

experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env

@@ -4,7 +4,6 @@
 #   --ep-size <TP*DP> (expert parallel; probes whether EP makes TP2/DP4 viable)
 #   EAGLE speculative decoding (steps=3, topk=1, draft=4)
 #   --chunked-prefill-size 8192 --max-prefill-tokens 16384
-#   --max-running-requests 64
 # Tests SGLang with three parallel configurations:
 #   TP=2, DP=4  -> 2 XPUs per replica, 4 replicas
 #   TP=4, DP=2  -> 4 XPUs per replica, 2 replicas
@@ -52,9 +51,8 @@ else
   )
 fi
 
-# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so
+# SGLang server settings, verified for P800 / DeepSeek-V4-Flash-INT8.
-# the context length is capped well below the H20 1M setting.
+CONTEXT_LENGTH="${CONTEXT_LENGTH:-65536}"
-CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}"
 MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
 
 # Official w8a8 parameters (from /workspace/ds_v4/w8a8/run_server.sh).
@@ -63,7 +61,6 @@ MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
 EP_SIZE="${EP_SIZE:-}"
 CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-8192}"
 MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-16384}"
-MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
 # EAGLE speculative decoding (official defaults: steps=3, topk=1, draft=4).
 SPEC_NUM_STEPS="${SPEC_NUM_STEPS:-3}"
 SPEC_EAGLE_TOPK="${SPEC_EAGLE_TOPK:-1}"

experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh

@@ -72,7 +72,7 @@ engine_build_server_args() {
   if (( dp == 1 )); then
     ep_args="--ep-size ${EP_SIZE:-$tp}"
   fi
-  local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
+  local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
   if (( dp > 1 )); then
     args="${args} --dp-size ${dp}"
   fi

experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh

@@ -72,7 +72,7 @@ engine_build_server_args() {
   if (( dp == 1 )); then
     ep_args="--ep-size ${EP_SIZE:-$tp}"
   fi
-  local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
+  local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
   if (( dp > 1 )); then
     args="${args} --dp-size ${dp}"
   fi

experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh

@@ -93,14 +93,13 @@ env_args=(
 # experiment) PLUS the official w8a8 run_server.sh parameters:
 #   EAGLE speculative decoding (steps/topk/draft from config.env)
 #   --chunked-prefill-size / --max-prefill-tokens (official long-prefill chunking)
-#   --max-running-requests 64 (official cap)
 # --ep-size is only added when DP=1: this vendor image crashes with
 # ZeroDivisionError when EP spans DP replicas (probed 2026-07-17).
 ep_args=""
 if [[ "$DP" -eq 1 ]]; then
   ep_args="--ep-size ${EP_SIZE:-$TP}"
 fi
-launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
+launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}"
 
 if [[ "$DP" -gt 1 ]]; then
   launch_args="${launch_args} --dp-size ${DP}"

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env

@@ -6,7 +6,7 @@
 # PLATEAU_PATIENCE consecutive points.
 
 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
-SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}"
 
 # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
 ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 # TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using SGLang.
-# Tests SGLang with three parallel configurations:
+# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache
-#   TP=2, DP=4  -> 2 GPUs per replica, 4 replicas
+# initialization on this machine.
+# Tests SGLang with two parallel configurations:
 #   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
 #   TP=8, DP=1  -> 8 GPUs, no data parallelism
 
@@ -28,19 +29,15 @@ RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
 
 # Parallel configurations to test. Format: "TP DP"
 declare -a PARALLEL_CONFIGS=(
-  "2 4"
   "4 2"
   "8 1"
 )
 
-# SGLang server settings (tuned for RTX 6000D 83GB)
+# SGLang server settings, verified for RTX 6000D / DeepSeek-V4-Flash.
-# Context length and scheduler capacity use SGLang's model/default settings.
+MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}"
-MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"
+MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
-MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"
+CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
-
+MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
-# RTX 6000D cannot capture DSV4 decode graphs at the default large batches.
-# This limits graph pre-capture only; it does not cap request concurrency.
-CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-16}"
 
 # Deployment switch. 0 = native sglang venv, 1 = Docker.
 USE_DOCKER="${USE_DOCKER:-1}"

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -73,15 +73,14 @@ engine_build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )
   if (( dp > 1 )); then
     args+=(--dp-size "$dp")
   fi
-  if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
-    args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
-  fi
   printf '%q ' "${args[@]}"
 }
 

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -73,15 +73,14 @@ engine_build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )
   if (( dp > 1 )); then
     args+=(--dp-size "$dp")
   fi
-  if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
-    args+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
-  fi
   printf '%q ' "${args[@]}"
 }
 

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh

@@ -95,6 +95,8 @@ build_server_args() {
     --tp-size "$tp"
     --moe-runner-backend "$MOE_RUNNER_BACKEND"
     --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
     --host 0.0.0.0
     --port "$SGLANG_PORT"
   )

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh

@@ -28,6 +28,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$PORT"
 )

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh

@@ -34,6 +34,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$PORT"
 )
@@ -44,10 +46,6 @@ if [[ "$DP" -gt 1 ]]; then
   )
 fi
 
-if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
-  SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
-fi
-
 SERVER_ARGS_STR="${SERVER_ARGS[*]}"
 
 echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="

experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_dp.sh

@@ -40,6 +40,8 @@ SERVER_ARGS=(
   --tp-size "$TP"
   --moe-runner-backend "$MOE_RUNNER_BACKEND"
   --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
   --host 0.0.0.0
   --port "$SGLANG_PORT"
 )
@@ -50,10 +52,6 @@ if [[ "$DP" -gt 1 ]]; then
   )
 fi
 
-if [[ -n "${CUDA_GRAPH_MAX_BS_DECODE:-}" ]]; then
-  SERVER_ARGS+=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE")
-fi
-
 SERVER_ARGS_STR="${SERVER_ARGS[*]}"
 
 echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ==="

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env

@@ -6,7 +6,7 @@
 # PLATEAU_PATIENCE consecutive points.
 
 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
-SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}"
 
 # At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
 ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env

@@ -32,11 +32,12 @@ declare -a PARALLEL_CONFIGS=(
   "8 1"
 )
 
-# vLLM server settings
+# vLLM server settings, verified for RTX 6000D / DeepSeek-V4-Flash.
-# Context length and scheduler capacity use vLLM's model/default settings.
 GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
 KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
 BLOCK_SIZE="${BLOCK_SIZE:-256}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
 
 # Deployment switch. 0 = native vllm venv, 1 = Docker.
 USE_DOCKER="${USE_DOCKER:-1}"

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -66,6 +66,8 @@ engine_build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh

@@ -66,6 +66,8 @@ engine_build_server_args() {
     --block-size "$BLOCK_SIZE"
     --tensor-parallel-size "$tp"
     --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
     --host 0.0.0.0
     --port "$ENGINE_PORT"
   )

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh

@@ -34,6 +34,8 @@ SERVER_ARGS=(
   --block-size "$BLOCK_SIZE"
   --tensor-parallel-size "$TP"
   --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
   --host 0.0.0.0
   --port "$PORT"
 )

experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh

@@ -39,6 +39,8 @@ SERVER_ARGS=(
   --block-size "$BLOCK_SIZE"
   --tensor-parallel-size "$TP"
   --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
   --host 0.0.0.0
   --port "$VLLM_PORT"
 )