[Feat] add 6000D DSV4 tiny adaptive benchmarks

2026-07-21 11:54:04 +08:00 · 2026-07-21 11:54:04 +08:00 · 0c2256ca1c
commit 0c2256ca1c
parent 039a2fa7b9
13 changed files with 1060 additions and 0 deletions
--- a/README.md
+++ b/README.md
@ -1,5 +1,9 @@
 # sskj — 多平台大模型推理性能基准测试项目

+> **更新（2026-07-21 11:54:30 +0800）**
+> - 新增 RTX 6000D / DeepSeek-V4-Flash tiny 自适应并发实验：输出固定 1K，输入遍历 1K、2K、4K、8K、16K、32K、64K、128K，并发采用 `C=16` 起、每次加 16、首点失败时按 `16 -> 8 -> 1` 回退。
+> - vLLM 覆盖 TP2/DP4、TP4/DP2、TP8/DP1，部署固定为 128K context、128 活跃请求、0.9 显存比例；SGLang 覆盖 TP4/DP2、TP8/DP1，固定为 128K、64、0.9，TP2/DP4 因 SM120 Marlin 权重加载 OOM 明确排除。
+
 > **更新（2026-07-20 17:55:20 +0800）**
 > - H20 SGLang TPxDP matrix 默认传入 `--cuda-graph-max-bs-decode 128`，使高并发 decode 在 batch 不超过 128 时持续使用 CUDA Graph；Docker 与 native 启动入口均已覆盖。

--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/adaptive_config.env
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/adaptive_config.env
@ -0,0 +1,56 @@
+# Adaptive concurrency search settings.
+#
+# For each fixed (TP, DP, ISL, OSL), probe:
+#   C = start, start * multiplier, ... up to max
+# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
+# PLATEAU_PATIENCE consecutive points.
+
+SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-64}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
+SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
+NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
+
+# A gain below 2% is treated as throughput saturation. Two consecutive
+# low-gain points prevent one noisy measurement from stopping the search.
+TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
+PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
+
+# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
+TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
+ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
+
+# Keep the same random workload semantics as the fixed matrix baseline.
+# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
+# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
+# random-ids to use generated token IDs without a ShareGPT seed dataset.
+BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
+# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
+# ISL/OSL points; lower values intentionally benchmark a length distribution.
+RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
+# Before each measured point, warm up with the same concurrency so lazy kernel
+# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
+# cap; set a positive cap only when very high-concurrency warmup is impractical.
+BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
+
+# Reject a point if the completed request count or actual token lengths do not
+# match the requested workload.
+INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
+OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
+
+MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
+SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Optional space-separated filters, useful for smoke tests:
+#   TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
+TP_LIST="${TP_LIST:-}"
+ISL_LIST="${ISL_LIST:-}"
+OSL_LIST="${OSL_LIST:-}"
+
+DRY_RUN="${DRY_RUN:-0}"
+# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
+GRID_LIMIT="${GRID_LIMIT:-0}"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/config.env
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/config.env
@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D
+# (8 GPUs) using SGLang. It fixes OSL=1024 and sweeps ISL=1K..128K.
+# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache
+# initialization on this machine.
+# Tests SGLang with two parallel configurations:
+#   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
+#   TP=8, DP=1  -> 8 GPUs, no data parallelism
+
+EXPERIMENT="dsv4_pro6000_sglang_tiny_1k_output"
+MODEL_NAME="DeepSeek-V4-Flash"
+MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
+SERVED_MODEL_NAME="deepseek-v4-flash"
+
+SGLANG_PORT="${SGLANG_PORT:-30031}"
+
+# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
+# and the benchmark client. Defaults to the system python3 if the sglang venv
+# does not exist on the host.
+VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
+
+# Run the benchmark client natively (0) or inside Docker (1).
+USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
+
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
+
+# Runtime working directory for logs, pid files, and tmp. Defaults to a local
+# directory under this experiment so the benchmark is self-contained.
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+
+# Parallel configurations to test. Format: "TP DP"
+declare -a PARALLEL_CONFIGS=(
+  "4 2"
+  "8 1"
+)
+
+# SGLang server settings, verified for this model and machine.
+MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}"
+MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
+CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
+MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
+
+# Deployment switch. 0 = native sglang venv, 1 = Docker.
+USE_DOCKER="${USE_DOCKER:-1}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
+
+# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
+BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
+DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
+SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
+CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_sm120_cache}"
+
+# Matrix contains ISL=1K..128K with fixed OSL=1K.
+MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
+MATRIX_MODE="${MATRIX_MODE:-Y}"
+
+# Sampling density for concurrency.
+# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
+# 2 = only test the low and high endpoints.
+export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
+
+# Per-scenario timeout to avoid hangs (seconds).
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+
+# GPU memory sampling interval (seconds).
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Dry-run mode: if 1, only log the server args and scenario plan without starting
+# any server or sending requests.
+DRY_RUN="${DRY_RUN:-0}"
+
+# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
+GRID_LIMIT="${GRID_LIMIT:-0}"
+
+# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
+# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
+# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
+PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/matrix.json
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/matrix.json
@ -0,0 +1,14 @@
+{
+  "comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.",
+  "mode": "Y",
+  "matrix": {
+    "1024": {"1024": "Y"},
+    "2048": {"1024": "Y"},
+    "4096": {"1024": "Y"},
+    "8192": {"1024": "Y"},
+    "16384": {"1024": "Y"},
+    "32768": {"1024": "Y"},
+    "65536": {"1024": "Y"},
+    "131072": {"1024": "Y"}
+  }
+}
--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/run_adaptive_concurrency_add16.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/run_adaptive_concurrency_add16.sh
@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="sglang"
+ENGINE_PORT="$SGLANG_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if [[ -n "${CONTAINER_NAME:-}" ]]; then
+      if docker exec "$CONTAINER_NAME" kill -0 "$pid" 2>/dev/null; then
+        log "stopping sglang in persistent container pid=${pid} tp=${tp} dp=${dp}"
+        docker exec "$CONTAINER_NAME" kill "$pid" 2>/dev/null || true
+        sleep 5
+        docker exec "$CONTAINER_NAME" kill -9 "$pid" 2>/dev/null || true
+      fi
+    elif kill -0 "$pid" 2>/dev/null; then
+      log "stopping sglang server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    docker rm -f "${EXPERIMENT}_sglang_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  fi
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    python3 -m sglang.launch_server --model-path "$MODEL_PATH"
+    --trust-remote-code
+    --tp-size "$tp"
+    --moe-runner-backend "$MOE_RUNNER_BACKEND"
+    --mem-fraction-static "$MEM_FRACTION_STATIC"
+    --context-length "$CONTEXT_LENGTH"
+    --max-running-requests "$MAX_RUNNING_REQUESTS"
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--dp-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  log "starting sglang server tp=${tp} dp=${dp}"
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    bash "${SCRIPT_DIR}/run_sglang_in_container.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  else
+    bash "${SCRIPT_DIR}/start_sglang_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  fi
+  if ! engine_is_healthy; then
+    log "ERROR: sglang health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  if [[ -z "${CONTAINER_NAME:-}" ]]; then
+    ACTIVE_ENGINE_SERVER_LOG="$(
+      find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+        -name "${EXPERIMENT}_sglang*tp${tp}_dp${dp}_*.log" \
+        -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+    )"
+  fi
+  log "sglang server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-container:/tmp/sglang_server.log}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  if grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null; then
+    return 0
+  fi
+  if [[ -n "${CONTAINER_NAME:-}" ]]; then
+    docker exec "$CONTAINER_NAME" grep -Eiq "$pattern" /tmp/sglang_server.log 2>/dev/null
+    return $?
+  fi
+  return 1
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local osl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend sglang
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$osl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
+    : # random-ids does not need --tokenize-prompt
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      --entrypoint python3 \
+      "$DOCKER_IMAGE" \
+      -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
+  else
+    "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+# If the initial concurrency violates the TTFT SLO, search downward. Stop at
+# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
+export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
+# When concurrency 1 still has a severely excessive TTFT, stop the remaining
+# shapes in this TP/DP group. Zero disables this rule.
+export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
+
+adaptive_main "$@"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_docker.sh
@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Start SGLang server in Docker for a given TPxDP configuration.
+# Usage: start_sglang_docker.sh <TP> <DP>
+#
+# Uses the verified SGLang SM120 image and keeps its JIT cache on
+# persistent storage. The container is removed automatically on stop.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
+
+IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
+PORT="${SGLANG_PORT:-30031}"
+NAME="${EXPERIMENT}_sglang_tp${TP}_dp${DP}"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+rm -f "$PID_FILE"
+
+# Clean up any stale container with the same name.
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+
+SERVER_ARGS=(
+  -m sglang.launch_server
+  --model-path "$MODEL_PATH"
+  --trust-remote-code
+  --tp-size "$TP"
+  --moe-runner-backend "$MOE_RUNNER_BACKEND"
+  --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
+  --host 0.0.0.0
+  --port "$PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --dp-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting SGLang server in Docker (TP=${TP}, DP=${DP}) ==="
+echo "Image: $IMAGE"
+echo "Model: $MODEL_PATH"
+echo "Container name: $NAME"
+echo "Host port: $PORT"
+echo "Command: python3 ${SERVER_ARGS_STR}"
+echo "Log: $LOG"
+
+# Run docker in the foreground so that killing the host process stops the
+# container (the --rm flag ensures cleanup). nohup lets us background it and
+# capture the host PID in the same way as the bare-metal start script.
+nohup docker run --rm \
+  --name "$NAME" \
+  --gpus all \
+  --privileged \
+  --ipc=host \
+  --network host \
+  --ulimit memlock=-1 \
+  --ulimit stack=67108864 \
+  --entrypoint python3 \
+  -v "${MODEL_PATH}:${MODEL_PATH}:ro" \
+  -v "${CACHE_DIR}:/root/.cache" \
+  -v "${RUNTIME_BASE}/tmp:/tmp" \
+  -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
+  -e PYTHONUNBUFFERED=1 \
+  -e HF_HUB_OFFLINE=1 \
+  -e TRANSFORMERS_OFFLINE=1 \
+  -e PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" \
+  "$IMAGE" \
+  "${SERVER_ARGS[@]}" \
+  > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${PORT}..."
+
+for i in $(seq 1 240); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
+    echo "SGLang server is ready at http://127.0.0.1:${PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  if ! kill -0 $PID 2>/dev/null; then
+    echo "ERROR: Docker SGLang server exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/240)"
+  sleep 5
+done
+
+echo "ERROR: Docker SGLang server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1
--- a/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tiny_1k_output/start_sglang_dp.sh
@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Start SGLang server for a given TP×DP configuration.
+# Usage: start_sglang_dp.sh <TP> <DP>
+#
+# By default this delegates to the Docker start script because the experiment
+# is intended to run SGLang inside a container. Set USE_DOCKER=0 to use the
+# local VENV_CLIENT environment instead.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+if [[ "${USE_DOCKER:-1}" == "1" ]]; then
+  exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@"
+fi
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
+
+VENV="${VENV_CLIENT}"
+export PATH="$VENV/bin:$PATH"
+export PYTHONUNBUFFERED=1
+export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
+export TMPDIR="${RUNTIME_BASE}/tmp"
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_sglang_tp${TP}_dp${DP}.pid"
+
+rm -f "$PID_FILE"
+
+SERVER_ARGS=(
+  python3 -m sglang.launch_server
+  --model-path "$MODEL_PATH"
+  --trust-remote-code
+  --tp-size "$TP"
+  --moe-runner-backend "$MOE_RUNNER_BACKEND"
+  --mem-fraction-static "$MEM_FRACTION_STATIC"
+  --context-length "$CONTEXT_LENGTH"
+  --max-running-requests "$MAX_RUNNING_REQUESTS"
+  --host 0.0.0.0
+  --port "$SGLANG_PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --dp-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting SGLang server (TP=${TP}, DP=${DP}) ==="
+echo "Model: $MODEL_PATH"
+echo "Port: $SGLANG_PORT"
+echo "Command: $SERVER_ARGS_STR"
+echo "Log: $LOG"
+
+nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${SGLANG_PORT}..."
+
+for i in $(seq 1 240); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then
+    echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  if ! kill -0 $PID 2>/dev/null; then
+    echo "ERROR: SGLang server exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/240)"
+  sleep 5
+done
+
+echo "ERROR: SGLang server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/adaptive_config.env
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/adaptive_config.env
@ -0,0 +1,56 @@
+# Adaptive concurrency search settings.
+#
+# For each fixed (TP, DP, ISL, OSL), probe:
+#   C = start, start * multiplier, ... up to max
+# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
+# PLATEAU_PATIENCE consecutive points.
+
+SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-128}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
+SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
+NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
+
+# A gain below 2% is treated as throughput saturation. Two consecutive
+# low-gain points prevent one noisy measurement from stopping the search.
+TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
+PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
+
+# Stop a shape when p95 TTFT exceeds the SLO; keep group skipping disabled.
+TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
+ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
+
+# Keep the same random workload semantics as the fixed matrix baseline.
+# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
+# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
+# random-ids to use generated token IDs without a ShareGPT seed dataset.
+BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
+# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
+# ISL/OSL points; lower values intentionally benchmark a length distribution.
+RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
+# Before each measured point, warm up with the same concurrency so lazy kernel
+# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
+# cap; set a positive cap only when very high-concurrency warmup is impractical.
+BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
+
+# Reject a point if the completed request count or actual token lengths do not
+# match the requested workload.
+INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
+OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
+
+MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
+SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Optional space-separated filters, useful for smoke tests:
+#   TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
+TP_LIST="${TP_LIST:-}"
+ISL_LIST="${ISL_LIST:-}"
+OSL_LIST="${OSL_LIST:-}"
+
+DRY_RUN="${DRY_RUN:-0}"
+# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
+GRID_LIMIT="${GRID_LIMIT:-0}"
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/config.env
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/config.env
@ -0,0 +1,78 @@
+# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D
+# (8 GPUs) using vLLM. It fixes OSL=1024 and sweeps ISL=1K..128K.
+# Tests vLLM with three parallel configurations:
+#   TP=2, DP=4  -> 2 GPUs per replica, 4 replicas
+#   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
+#   TP=8, DP=1  -> 8 GPUs, no data parallelism
+
+EXPERIMENT="dsv4_pro6000_vllm_tiny_1k_output"
+MODEL_NAME="DeepSeek-V4-Flash"
+MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
+SERVED_MODEL_NAME="deepseek-v4-flash"
+
+VLLM_PORT="${VLLM_PORT:-30030}"
+
+# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
+# and the benchmark client. Defaults to the system python3 if the sglang venv
+# does not exist on the host.
+VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
+
+# Run the benchmark client natively (0) or inside Docker (1).
+USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
+
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
+
+# Runtime working directory for logs, pid files, and tmp. Defaults to a local
+# directory under this experiment so the benchmark is self-contained.
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+
+# Parallel configurations to test. Format: "TP DP"
+declare -a PARALLEL_CONFIGS=(
+  "2 4"
+  "4 2"
+  "8 1"
+)
+
+# vLLM server settings, verified for this model and machine.
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
+BLOCK_SIZE="${BLOCK_SIZE:-256}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
+
+# Deployment switch. 0 = native vllm venv, 1 = Docker.
+USE_DOCKER="${USE_DOCKER:-1}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
+
+# Benchmark client Docker image. vLLM's image does not include the SGLang benchmark client,
+# so the client runs inside the SGLang image and targets the vLLM backend via the
+# OpenAI-compatible API.
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
+
+# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
+BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
+DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
+SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
+CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/vllm_sm120_cache}"
+
+# Matrix contains ISL=1K..128K with fixed OSL=1K.
+MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
+MATRIX_MODE="${MATRIX_MODE:-Y}"
+
+# Sampling density for concurrency.
+# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
+# 2 = only test the low and high endpoints.
+export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
+
+# Per-scenario timeout to avoid hangs (seconds).
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+
+# GPU memory sampling interval (seconds).
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Dry-run mode: if 1, only log the server args and scenario plan without starting
+# any server or sending requests.
+DRY_RUN="${DRY_RUN:-0}"
+
+# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
+GRID_LIMIT="${GRID_LIMIT:-0}"
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/matrix.json
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/matrix.json
@ -0,0 +1,14 @@
+{
+  "comment": "Tiny adaptive test: fixed OSL=1024, ISL=1K..128K. Only Y entries are run.",
+  "mode": "Y",
+  "matrix": {
+    "1024": {"1024": "Y"},
+    "2048": {"1024": "Y"},
+    "4096": {"1024": "Y"},
+    "8192": {"1024": "Y"},
+    "16384": {"1024": "Y"},
+    "32768": {"1024": "Y"},
+    "65536": {"1024": "Y"},
+    "131072": {"1024": "Y"}
+  }
+}
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/run_adaptive_concurrency_add16.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/run_adaptive_concurrency_add16.sh
@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+# Tiny test: find Total-TPS saturation for fixed OSL=1K and ISL=1K..128K.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="vllm"
+ENGINE_PORT="$VLLM_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if kill -0 "$pid" 2>/dev/null; then
+      log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    vllm serve "$MODEL_PATH"
+    --trust-remote-code
+    --kv-cache-dtype "$KV_CACHE_DTYPE"
+    --block-size "$BLOCK_SIZE"
+    --tensor-parallel-size "$tp"
+    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--data-parallel-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  log "starting vllm server tp=${tp} dp=${dp}"
+  bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  if ! engine_is_healthy; then
+    log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  ACTIVE_ENGINE_SERVER_LOG="$(
+    find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+      -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
+      -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+  )"
+  log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local osl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend vllm
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$osl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  elif [[ "$BENCH_DATASET_NAME" == "random-ids" ]]; then
+    : # random-ids does not need --tokenize-prompt
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      -e HF_HUB_OFFLINE=1 \
+      -e TRANSFORMERS_OFFLINE=1 \
+      --entrypoint python3 \
+      "$DOCKER_CLIENT_IMAGE" \
+      -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
+  else
+    "$PYTHON" -m "$SGLANG_BENCH_MODULE" "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON SGLANG_BENCH_MODULE
+
+export SEARCH_START_CONCURRENCY=16
+export SEARCH_ADDEND=16
+# If the initial concurrency violates the TTFT SLO, search downward. Stop at
+# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
+export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
+# When concurrency 1 still has a severely excessive TTFT, stop the remaining
+# shapes in this TP/DP group. Zero disables this rule.
+export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
+
+adaptive_main "$@"
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_docker.sh
@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Start vLLM server in Docker for a given TPxDP configuration.
+# Usage: start_vllm_docker.sh <TP> <DP>
+#
+# Uses the verified vLLM SM120 image and keeps its cache on persistent
+# storage. The container is removed automatically on stop.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" "$CACHE_DIR"
+
+IMAGE="${DOCKER_IMAGE:-vllm-sm120-dsv4:0.25.1-fi0.6.14}"
+PORT="${VLLM_PORT:-30030}"
+NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+rm -f "$PID_FILE"
+
+# Clean up any stale container with the same name.
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+
+SERVER_ARGS=(
+  serve "$MODEL_PATH"
+  --trust-remote-code
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
+  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
+  --host 0.0.0.0
+  --port "$PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --data-parallel-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
+echo "Image: $IMAGE"
+echo "Model: $MODEL_PATH"
+echo "Container name: $NAME"
+echo "Host port: $PORT"
+echo "Command: vllm ${SERVER_ARGS_STR}"
+echo "Log: $LOG"
+
+# Run docker in the foreground so that killing the host process stops the
+# container (the --rm flag ensures cleanup). nohup lets us background it and
+# capture the host PID in the same way as the bare-metal start script.
+nohup docker run --rm \
+  --name "$NAME" \
+  --gpus all \
+  --privileged \
+  --ipc=host \
+  --network host \
+  --ulimit memlock=-1 \
+  --ulimit stack=67108864 \
+  --entrypoint vllm \
+  -v "${MODEL_PATH}:${MODEL_PATH}:ro" \
+  -v "${CACHE_DIR}:/root/.cache" \
+  -v "${RUNTIME_BASE}/tmp:/tmp" \
+  -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
+  -e PYTHONUNBUFFERED=1 \
+  -e HF_HUB_OFFLINE=1 \
+  -e TRANSFORMERS_OFFLINE=1 \
+  "$IMAGE" \
+  "${SERVER_ARGS[@]}" \
+  > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${PORT}..."
+
+for i in $(seq 1 600); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
+    echo "vLLM server is ready at http://127.0.0.1:${PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  # Check if the Docker container is still running (not the nohup PID).
+  # Allow a brief grace period for the container to appear in docker ps.
+  container_running=0
+  for _ in $(seq 1 3); do
+    if docker ps --filter "name=${NAME}" --format '{{.Names}}' | grep -q "^${NAME}$"; then
+      container_running=1
+      break
+    fi
+    sleep 1
+  done
+  if [[ "$container_running" -eq 0 ]]; then
+    echo "ERROR: Docker vLLM container exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/600)"
+  sleep 5
+done
+
+echo "ERROR: Docker vLLM server not healthy after 600 retries"
+tail -200 "$LOG"
+exit 1
--- a/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tiny_1k_output/start_vllm_dp.sh
@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Start vLLM server for a given TP×DP configuration.
+# Usage: start_vllm_dp.sh <TP> <DP>
+#
+# By default this delegates to the Docker start script because the experiment
+# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the
+# local VENV_VLLM environment instead.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+if [[ "${USE_DOCKER:-1}" == "1" ]]; then
+  exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@"
+fi
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
+
+VENV="${VENV_VLLM}"
+export PATH="$VENV/bin:$PATH"
+export PYTHONUNBUFFERED=1
+export TMPDIR="${RUNTIME_BASE}/tmp"
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+
+rm -f "$PID_FILE"
+
+SERVER_ARGS=(
+  vllm serve "$MODEL_PATH"
+  --trust-remote-code
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
+  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
+  --host 0.0.0.0
+  --port "$VLLM_PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --data-parallel-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ==="
+echo "Model: $MODEL_PATH"
+echo "Port: $VLLM_PORT"
+echo "Command: $SERVER_ARGS_STR"
+echo "Log: $LOG"
+
+nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${VLLM_PORT}..."
+
+for i in $(seq 1 600); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
+    echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  if ! kill -0 $PID 2>/dev/null; then
+    echo "ERROR: vLLM server exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/600)"
+  sleep 5
+done
+
+echo "ERROR: vLLM server not healthy after 600 retries"
+tail -200 "$LOG"
+exit 1