vllm tp_dp_matrix: dockerize deployment and fix missing DOCKER_CLIENT_IMAGE export

- Run vLLM server inside vllm/vllm-openai:latest container on H200. - Run benchmark client inside lmsysorg/sglang:latest using sglang.bench_serving --backend vllm, matching the sglang reference. - Use RUNTIME_BASE for self-contained logs/pids/tmp instead of hardcoded /data/user1/yy paths. - Update model path to /data3/hf_models/DeepSeek-V4-Flash. - Sample only low/high concurrency endpoints to match sglang matrix. - Fix docker 'invalid reference format' caused by DOCKER_CLIENT_IMAGE not being exported to bash -c subshells used by run_bench_serving. - Add offline dummy ShareGPT dataset for random benchmark mode.
2026-07-10 08:44:38 +00:00 · 2026-07-10 08:44:38 +00:00 · d783143ddb
commit d783143ddb
parent 4d5b6a77ca
6 changed files with 177 additions and 103 deletions
--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py
@ -22,10 +22,10 @@ def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
    ttft_ok = ttft_p95_ms < ttft_limit_ms
    tpot_ok = tpot_mean_ms < tpot_limit_ms
    if ttft_ok and tpot_ok:
-        return "✅"
+        return "PASS"
    if ttft_ok or tpot_ok:
-        return "⚠️"
-    return "❌"
+        return "PARTIAL"
+    return "FAIL"


 def gpu_memory_str(gpu: dict | None) -> str:
@ -82,7 +82,7 @@ def main():
        f.write("## Summary\n\n")
        f.write(f"- Model: `{model}`\n")
        f.write(f"- Hardware: {hardware}\n")
-        f.write("- Backend: vLLM\n")
+        f.write("- Backend: vLLM (Docker)\n")
        f.write("- Benchmark client: `sglang.bench_serving`\n")
        f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")

@ -152,7 +152,7 @@ def main():

        f.write("\n## Notes\n\n")
        f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
-        f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
+        f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
        f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
        f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")

--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env
@ -1,4 +1,4 @@
-# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs).
+# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using vLLM.
 # Tests vLLM with three parallel configurations:
 #   TP=2, DP=4  -> 2 GPUs per replica, 4 replicas
 #   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
@ -6,18 +6,25 @@

 EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix"
 MODEL_NAME="DeepSeek-V4-Flash"
-MODEL_PATH="/data/models/DeepSeek-V4-Flash"
+MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash"
 SERVED_MODEL_NAME="deepseek-v4-flash"

 VLLM_PORT="${VLLM_PORT:-30030}"

-VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
-VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
-# The benchmark client is sglang.bench_serving, even when the backend is vLLM.
-VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
+# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
+# and the benchmark client. Defaults to the system python3 if the sglang venv
+# does not exist on the host.
+VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
+
+# Run the benchmark client natively (0) or inside Docker (1).
+USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"

 export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"

+# Runtime working directory for logs, pid files, and tmp. Defaults to a local
+# directory under this experiment so the benchmark is self-contained.
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+
 # Parallel configurations to test. Format: "TP DP"
 declare -a PARALLEL_CONFIGS=(
  "2 4"
@ -25,14 +32,34 @@ declare -a PARALLEL_CONFIGS=(
  "8 1"
 )

+# vLLM server settings
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
+BLOCK_SIZE="${BLOCK_SIZE:-256}"
+
+# Deployment switch. 0 = native vllm venv, 1 = Docker.
+USE_DOCKER="${USE_DOCKER:-1}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+
+# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
+# so the client runs inside the SGLang image and targets the vLLM backend via the
+# OpenAI-compatible API.
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
+
+# Dataset used by sglang.bench_serving --dataset-name random.
+# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
+# downloading from HuggingFace, which usually fails on offline H200 nodes.
+DATASET_PATH="${DATASET_PATH:-${SCRIPT_DIR}/runtime/dummy_sharegpt.json}"
+
 # Matrix and concurrency rules are defined in matrix.json by default.
-# This file is consumed by generate_scenarios.py.
 MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
 MATRIX_MODE="${MATRIX_MODE:-Y}"

 # Sampling density for concurrency.
 # 0 = use the default heuristic in generate_scenarios.py (6-8 points).
-# 2  = only test the two endpoints (low and high).
+# 2 = only test the low and high endpoints.
 export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"

 # Per-scenario timeout to avoid hangs (seconds).
@ -47,8 +74,3 @@ DRY_RUN="${DRY_RUN:-0}"

 # Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
 GRID_LIMIT="${GRID_LIMIT:-0}"
-
-# Docker deployment switch. If 1, vLLM is launched via Docker using the image
-# below instead of the local VENV_VLLM environment.
-USE_DOCKER="${USE_DOCKER:-0}"
-DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py
@ -15,23 +15,16 @@ from pathlib import Path


 def sample_concurrency(low: int, high: int, target: int) -> list[int]:
-    """Return a sorted list of concurrency values in [low, high].
+    """Return only the low and high concurrency values in [low, high].

-    If the range is small, return every integer. Otherwise sample roughly
-    `target` points linearly between low and high (inclusive).
+    For the TP×DP matrix we only need the two endpoints of the concurrency
+    range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for
+    API compatibility but is ignored.
    """
    assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
-    if high - low + 1 <= target:
-        return list(range(low, high + 1))
-
-    points = set()
-    points.add(low)
-    points.add(high)
-    step = (high - low) / (target - 1)
-    for i in range(1, target - 1):
-        v = low + round(step * i)
-        points.add(max(low, min(high, v)))
-    return sorted(points)
+    if low == high:
+        return [low]
+    return [low, high]


 def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh
@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM.
+# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM (Docker).
 set -Eeuo pipefail

 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@ -12,6 +12,10 @@ source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
 # shellcheck source=/dev/null
 source "${SCRIPT_DIR}/config.env"

+# Export variables used inside functions that are called via bash -c subshells.
+export DATASET_PATH MODEL_PATH RESULT_BASE \
+  DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
+
 RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
 RESULT_BASE="${SCRIPT_DIR}/results"
 MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
@ -21,7 +25,14 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
 DRY_RUN="${DRY_RUN:-0}"
 GRID_LIMIT="${GRID_LIMIT:-0}"

+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"

 log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
 mkdir -p "$log_dir_global"
@ -42,7 +53,7 @@ stop_server() {
  local tp="$1"
  local dp="$2"

-  local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
  if [[ -f "$pid_file" ]]; then
    local pid
    pid="$(cat "$pid_file")"
@ -55,6 +66,9 @@ stop_server() {
    rm -f "$pid_file"
  fi

+  # Fallback: remove any Docker container started by this experiment.
+  docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+
  # Fallback: kill any vllm serve processes for this model.
  pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
  pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true
@ -68,10 +82,14 @@ build_server_args() {
  local args=(
    "vllm serve" "$MODEL_PATH"
    --trust-remote-code
-    --kv-cache-dtype fp8
-    --block-size 256
+    --kv-cache-dtype "$KV_CACHE_DTYPE"
+    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
+    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
    --no-enable-flashinfer-autotune
+    --host 0.0.0.0
    --port "$VLLM_PORT"
  )
  if [[ "$dp" -gt 1 ]]; then
@ -106,20 +124,51 @@ restart_server() {
  start_server "$tp" "$dp"
 }

+run_bench_serving() {
+  # Run sglang.bench_serving either natively or inside the SGLang Docker image.
+  if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then
+    local vol_args=()
+    vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro")
+    if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then
+      vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}")
+    docker run --rm \
+      --network host \
+      "${vol_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      -e HF_HUB_OFFLINE=1 \
+      -e TRANSFORMERS_OFFLINE=1 \
+      -e HF_DATASETS_OFFLINE=1 \
+      "${DOCKER_CLIENT_IMAGE}" \
+      python -m sglang.bench_serving "$@"
+  else
+    "$PYTHON" -m sglang.bench_serving "$@"
+  fi
+}
+export -f run_bench_serving
+
 run_warmup() {
  local input_len="$1"
  local output_len="$2"

  log "warming up (input=${input_len}, output=${output_len}, num=1)"
-  "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
+  bash -c '
+    run_bench_serving \
      --backend vllm \
      --host 127.0.0.1 \
-    --port "$VLLM_PORT" \
-    --input-len "$input_len" \
-    --output-len "$output_len" \
-    --num 1 \
-    --env-python "$PYTHON" \
-    >> "${log_dir_global}/warmup.log" 2>&1
+      --port "'"$VLLM_PORT"'" \
+      --dataset-name random \
+      --dataset-path "'"$DATASET_PATH"'" \
+      --random-input-len "'"$input_len"'" \
+      --random-output-len "'"$output_len"'" \
+      --num-prompts 1 \
+      --max-concurrency 1 \
+      --request-rate 10000 \
+      --output-file /dev/null \
+      --output-details \
+      >> "'"${log_dir_global}/warmup.log"'" 2>&1
+  '
  log "warmup completed"
 }

@ -307,7 +356,7 @@ run_parallel_config() {
    "$ACCELERATOR" \
    "$CHIP" \
    "experiments/${EXPERIMENT_NAME}/run_bench.sh" \
-    "$VENV_VLLM" \
+    "$DOCKER_IMAGE" \
    "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash"

  local server_args_str
@ -383,20 +432,22 @@ run_parallel_config() {
    gpu_pid="$(start_gpu_monitor "$gpu_csv")"

    bench_rc=0
-    timeout "$SCENARIO_TIMEOUT_S" \
-      "$PYTHON" -m sglang.bench_serving \
+    timeout "$SCENARIO_TIMEOUT_S" bash -c '
+      run_bench_serving \
        --backend vllm \
        --host 127.0.0.1 \
-        --port "$VLLM_PORT" \
+        --port "'"$VLLM_PORT"'" \
        --dataset-name random \
-        --random-input-len "$isl" \
-        --random-output-len "$dsl" \
-        --num-prompts "$num" \
-        --max-concurrency "$conc" \
+        --dataset-path "'"$DATASET_PATH"'" \
+        --random-input-len "'"$isl"'" \
+        --random-output-len "'"$dsl"'" \
+        --num-prompts "'"$num"'" \
+        --max-concurrency "'"$conc"'" \
        --request-rate 10000 \
-        --output-file "$output_file" \
+        --output-file "'"$output_file"'" \
        --output-details \
-        > "$detail_log" 2>&1 || bench_rc=$?
+        > "'"$detail_log"'" 2>&1
+    ' || bench_rc=$?

    stop_gpu_monitor "$gpu_pid"

--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_docker.sh
@ -2,7 +2,7 @@
 # Start vLLM server in Docker for a given TPxDP configuration.
 # Usage: start_vllm_docker.sh <TP> <DP>
 #
-# Uses the vllm/vllm-openai image and the same minimal argument set as the
+# Uses the vllm/vllm-openai image and the same argument set as the
 # bare-metal start script. The container is removed automatically on stop.
 set -e

@ -13,15 +13,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${SCRIPT_DIR}/config.env"

-cd /data/user1/yy
-mkdir -p logs
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"

 IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
 PORT="${VLLM_PORT:-30030}"
 NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
-PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"

-LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
 rm -f "$PID_FILE"

 # Clean up any stale container with the same name.
@ -30,10 +30,14 @@ docker rm -f "$NAME" >/dev/null 2>&1 || true
 SERVER_ARGS=(
  serve "$MODEL_PATH"
  --trust-remote-code
-  --kv-cache-dtype bfloat16
-  --block-size 256
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
  --no-enable-flashinfer-autotune
+  --host 0.0.0.0
  --port "$PORT"
 )

@ -50,7 +54,7 @@ echo "Image: $IMAGE"
 echo "Model: $MODEL_PATH"
 echo "Container name: $NAME"
 echo "Host port: $PORT"
-echo "Command: vllm $SERVER_ARGS_STR"
+echo "Command: vllm ${SERVER_ARGS_STR}"
 echo "Log: $LOG"

 # Run docker in the foreground so that killing the host process stops the
@ -59,12 +63,17 @@ echo "Log: $LOG"
 nohup docker run --rm \
  --name "$NAME" \
  --gpus all \
+  --ipc host \
+  --shm-size 16g \
+  --ulimit memlock=-1 \
  --entrypoint vllm \
  -p "${PORT}:${PORT}" \
-  -v /data/models:/data/models:ro \
-  -v /data/user1/yy/tmp:/tmp \
+  -v "${MODEL_PATH}:${MODEL_PATH}:ro" \
+  -v "${RUNTIME_BASE}/tmp:/tmp" \
  -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
  -e PYTHONUNBUFFERED=1 \
+  -e HF_HUB_OFFLINE=1 \
+  -e TRANSFORMERS_OFFLINE=1 \
  "$IMAGE" \
  "${SERVER_ARGS[@]}" \
  > "$LOG" 2>&1 &
--- a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh
+++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh
@ -2,14 +2,9 @@
 # Start vLLM server for a given TP×DP configuration.
 # Usage: start_vllm_dp.sh <TP> <DP>
 #
-# Uses the minimal argument set from the official DeepSeek-V4-Flash recipe:
-#   --trust-remote-code
-#   --kv-cache-dtype fp8
-#   --block-size 256
-#   --tensor-parallel-size <TP>
-#   --no-enable-flashinfer-autotune
-# When DP > 1, only --data-parallel-size is added so vLLM uses its internal
-# DP load balancer (single API server, single port).
+# By default this delegates to the Docker start script because the experiment
+# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the
+# local VENV_VLLM environment instead.
 set -e

 TP="${1}"
@ -19,31 +14,35 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${SCRIPT_DIR}/config.env"

-if [[ "${USE_DOCKER:-0}" == "1" ]]; then
+if [[ "${USE_DOCKER:-1}" == "1" ]]; then
  exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@"
 fi

-cd /data/user1/yy
-mkdir -p logs
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"

 VENV="${VENV_VLLM}"
 export PATH="$VENV/bin:$PATH"
 export PYTHONUNBUFFERED=1
-export TMPDIR=/data/user1/yy/tmp
+export TMPDIR="${RUNTIME_BASE}/tmp"
 export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"

-LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
-PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"

 rm -f "$PID_FILE"

 SERVER_ARGS=(
  vllm serve "$MODEL_PATH"
  --trust-remote-code
-    --kv-cache-dtype fp8
-    --block-size 256
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
  --no-enable-flashinfer-autotune
+  --host 0.0.0.0
  --port "$VLLM_PORT"
 )