Add dsv4_pro6000_vllm_tp_dp_matrix experiment for RTX 6000D

- vLLM adaptive concurrency benchmark adapted from H200 version
- Key changes:
  - Model path: /data/hf_models/DeepSeek-V4-Flash
  - Venv path: /root/.miniconda3/envs/sglang
  - Dataset path: /data/yy/sskj/datasets/
  - DSL renamed to OSL (Output Sequence Length)
- Includes: config.env, adaptive_config.env, run_adaptive_concurrency.sh,
  start_vllm_dp.sh, start_vllm_docker.sh

experiments/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env

@@ -0,0 +1,49 @@
+# Adaptive concurrency search settings.
+#
+# For each fixed (TP, DP, ISL, OSL), probe:
+#   C = start, start * multiplier, ... up to max
+# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
+# PLATEAU_PATIENCE consecutive points.
+
+SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
+SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
+NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
+
+# A gain below 2% is treated as throughput saturation. Two consecutive
+# low-gain points prevent one noisy measurement from stopping the search.
+TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
+PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
+
+# Keep the same random workload semantics as the fixed matrix baseline.
+# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
+# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
+# random-ids to use generated token IDs without a ShareGPT seed dataset.
+BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
+# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
+# ISL/OSL points; lower values intentionally benchmark a length distribution.
+RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
+# Before each measured point, warm up with the same concurrency so lazy kernel
+# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
+# cap; set a positive cap only when very high-concurrency warmup is impractical.
+BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
+
+# Reject a point if the completed request count or actual token lengths do not
+# match the requested workload.
+INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
+OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
+
+MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
+SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Optional space-separated filters, useful for smoke tests:
+#   TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
+TP_LIST="${TP_LIST:-}"
+ISL_LIST="${ISL_LIST:-}"
+OSL_LIST="${OSL_LIST:-}"
+
+DRY_RUN="${DRY_RUN:-0}"
+# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
+GRID_LIMIT="${GRID_LIMIT:-0}"

experiments/dsv4_pro6000_vllm_tp_dp_matrix/config.env

@@ -0,0 +1,76 @@
+# TP×DP matrix experiment for DeepSeek-V4-Flash on RTX 6000D (8 GPUs) using vLLM.
+# Tests vLLM with three parallel configurations:
+#   TP=2, DP=4  -> 2 GPUs per replica, 4 replicas
+#   TP=4, DP=2  -> 4 GPUs per replica, 2 replicas
+#   TP=8, DP=1  -> 8 GPUs, no data parallelism
+
+EXPERIMENT="dsv4_pro6000_vllm_tp_dp_matrix"
+MODEL_NAME="DeepSeek-V4-Flash"
+MODEL_PATH="/data/hf_models/DeepSeek-V4-Flash"
+SERVED_MODEL_NAME="deepseek-v4-flash"
+
+VLLM_PORT="${VLLM_PORT:-30030}"
+
+# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
+# and the benchmark client. Defaults to the system python3 if the sglang venv
+# does not exist on the host.
+VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
+
+# Run the benchmark client natively (0) or inside Docker (1).
+USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
+
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
+
+# Runtime working directory for logs, pid files, and tmp. Defaults to a local
+# directory under this experiment so the benchmark is self-contained.
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+
+# Parallel configurations to test. Format: "TP DP"
+declare -a PARALLEL_CONFIGS=(
+  "2 4"
+  "4 2"
+  "8 1"
+)
+
+# vLLM server settings
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
+GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
+KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
+BLOCK_SIZE="${BLOCK_SIZE:-256}"
+
+# Deployment switch. 0 = native vllm venv, 1 = Docker.
+USE_DOCKER="${USE_DOCKER:-1}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+
+# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
+# so the client runs inside the SGLang image and targets the vLLM backend via the
+# OpenAI-compatible API.
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
+
+# Dataset used by sglang.bench_serving --dataset-name random.
+# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
+# downloading from HuggingFace, which usually fails on offline nodes.
+DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
+
+# Matrix and concurrency rules are defined in matrix.json by default.
+MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
+MATRIX_MODE="${MATRIX_MODE:-Y}"
+
+# Sampling density for concurrency.
+# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
+# 2 = only test the low and high endpoints.
+export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
+
+# Per-scenario timeout to avoid hangs (seconds).
+SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
+
+# GPU memory sampling interval (seconds).
+GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
+
+# Dry-run mode: if 1, only log the server args and scenario plan without starting
+# any server or sending requests.
+DRY_RUN="${DRY_RUN:-0}"
+
+# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
+GRID_LIMIT="${GRID_LIMIT:-0}"

experiments/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/adaptive_config.env"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh"
+
+ENGINE="vllm"
+ENGINE_PORT="$VLLM_PORT"
+RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
+ACTIVE_ENGINE_SERVER_LOG=""
+
+if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
+  PYTHON="${VENV_CLIENT}/bin/python"
+else
+  PYTHON="$(command -v python3)"
+fi
+
+DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
+
+engine_is_healthy() {
+  curl --fail --silent --show-error --max-time 5 \
+    "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
+}
+
+engine_stop_server() {
+  local tp="$1"
+  local dp="$2"
+  local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
+
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if kill -0 "$pid" 2>/dev/null; then
+      log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
+  ACTIVE_ENGINE_SERVER_LOG=""
+  sleep 2
+}
+
+engine_build_server_args() {
+  local tp="$1"
+  local dp="$2"
+  local -a args=(
+    vllm serve "$MODEL_PATH"
+    --trust-remote-code
+    --kv-cache-dtype "$KV_CACHE_DTYPE"
+    --block-size "$BLOCK_SIZE"
+    --tensor-parallel-size "$tp"
+    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+    --max-model-len "$MAX_MODEL_LEN"
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --no-enable-flashinfer-autotune
+    --host 0.0.0.0
+    --port "$ENGINE_PORT"
+  )
+  if (( dp > 1 )); then
+    args+=(--data-parallel-size "$dp")
+  fi
+  printf '%q ' "${args[@]}"
+}
+
+engine_start_server() {
+  local tp="$1"
+  local dp="$2"
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  log "starting vllm server tp=${tp} dp=${dp}"
+  bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
+  if ! engine_is_healthy; then
+    log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
+    return 1
+  fi
+  ACTIVE_ENGINE_SERVER_LOG="$(
+    find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
+      -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
+      -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
+  )"
+  log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
+}
+
+engine_detect_oom() {
+  local detail_log="$1"
+  local tp="$2"
+  local dp="$3"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
+  local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
+  local -a logs=("$detail_log" "$outer_log")
+  if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
+    logs+=("$ACTIVE_ENGINE_SERVER_LOG")
+  fi
+  grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
+}
+
+engine_run_bench() {
+  local isl="$1"
+  local osl="$2"
+  local concurrency="$3"
+  local num_prompts="$4"
+  local output_file="$5"
+  local warmup_requests
+  warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
+  local -a bench_args=(
+    --backend vllm
+    --host 127.0.0.1
+    --port "$ENGINE_PORT"
+    --dataset-name "$BENCH_DATASET_NAME"
+    --random-input-len "$isl"
+    --random-output-len "$osl"
+    --random-range-ratio "$RANDOM_RANGE_RATIO"
+    --num-prompts "$num_prompts"
+    --max-concurrency "$concurrency"
+    --request-rate 10000
+    --warmup-requests "$warmup_requests"
+    --output-file "$output_file"
+    --output-details
+    --disable-tqdm
+  )
+  if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+    bench_args+=(--dataset-path "$DATASET_PATH")
+  else
+    bench_args+=(--tokenize-prompt)
+  fi
+
+  if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
+    local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
+    if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
+      volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
+    fi
+    docker run --rm \
+      --network host \
+      "${volume_args[@]}" \
+      -e PYTHONUNBUFFERED=1 \
+      -e HF_HUB_OFFLINE=1 \
+      -e TRANSFORMERS_OFFLINE=1 \
+      "$DOCKER_CLIENT_IMAGE" \
+      python -m sglang.bench_serving "${bench_args[@]}"
+  else
+    "$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
+  fi
+}
+
+export -f engine_run_bench
+export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
+export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
+
+adaptive_main "$@"

experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Start vLLM server in Docker for a given TPxDP configuration.
+# Usage: start_vllm_docker.sh <TP> <DP>
+#
+# Uses the vllm/vllm-openai image and the same argument set as the
+# bare-metal start script. The container is removed automatically on stop.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
+
+IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
+PORT="${VLLM_PORT:-30030}"
+NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+rm -f "$PID_FILE"
+
+# Clean up any stale container with the same name.
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+
+SERVER_ARGS=(
+  serve "$MODEL_PATH"
+  --trust-remote-code
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
+  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
+  --no-enable-flashinfer-autotune
+  --host 0.0.0.0
+  --port "$PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --data-parallel-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
+echo "Image: $IMAGE"
+echo "Model: $MODEL_PATH"
+echo "Container name: $NAME"
+echo "Host port: $PORT"
+echo "Command: vllm ${SERVER_ARGS_STR}"
+echo "Log: $LOG"
+
+# Run docker in the foreground so that killing the host process stops the
+# container (the --rm flag ensures cleanup). nohup lets us background it and
+# capture the host PID in the same way as the bare-metal start script.
+nohup docker run --rm \
+  --name "$NAME" \
+  --gpus all \
+  --ipc host \
+  --shm-size 16g \
+  --ulimit memlock=-1 \
+  --entrypoint vllm \
+  -p "${PORT}:${PORT}" \
+  -v "${MODEL_PATH}:${MODEL_PATH}:ro" \
+  -v "${RUNTIME_BASE}/tmp:/tmp" \
+  -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
+  -e PYTHONUNBUFFERED=1 \
+  -e HF_HUB_OFFLINE=1 \
+  -e TRANSFORMERS_OFFLINE=1 \
+  "$IMAGE" \
+  "${SERVER_ARGS[@]}" \
+  > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${PORT}..."
+
+for i in $(seq 1 240); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
+    echo "vLLM server is ready at http://127.0.0.1:${PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  if ! kill -0 $PID 2>/dev/null; then
+    echo "ERROR: Docker vLLM server exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/240)"
+  sleep 5
+done
+
+echo "ERROR: Docker vLLM server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1

experiments/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Start vLLM server for a given TP×DP configuration.
+# Usage: start_vllm_dp.sh <TP> <DP>
+#
+# By default this delegates to the Docker start script because the experiment
+# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the
+# local VENV_VLLM environment instead.
+set -e
+
+TP="${1}"
+DP="${2}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+if [[ "${USE_DOCKER:-1}" == "1" ]]; then
+  exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@"
+fi
+
+RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
+mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
+
+VENV="${VENV_VLLM}"
+export PATH="$VENV/bin:$PATH"
+export PYTHONUNBUFFERED=1
+export TMPDIR="${RUNTIME_BASE}/tmp"
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
+
+LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
+
+rm -f "$PID_FILE"
+
+SERVER_ARGS=(
+  vllm serve "$MODEL_PATH"
+  --trust-remote-code
+  --kv-cache-dtype "$KV_CACHE_DTYPE"
+  --block-size "$BLOCK_SIZE"
+  --tensor-parallel-size "$TP"
+  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
+  --max-model-len "$MAX_MODEL_LEN"
+  --max-num-seqs "$MAX_NUM_SEQS"
+  --no-enable-flashinfer-autotune
+  --host 0.0.0.0
+  --port "$VLLM_PORT"
+)
+
+if [[ "$DP" -gt 1 ]]; then
+  SERVER_ARGS+=(
+    --data-parallel-size "$DP"
+  )
+fi
+
+SERVER_ARGS_STR="${SERVER_ARGS[*]}"
+
+echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ==="
+echo "Model: $MODEL_PATH"
+echo "Port: $VLLM_PORT"
+echo "Command: $SERVER_ARGS_STR"
+echo "Log: $LOG"
+
+nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health on port ${VLLM_PORT}..."
+
+for i in $(seq 1 240); do
+  if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
+    echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
+    echo "Log: $LOG"
+    exit 0
+  fi
+  if ! kill -0 $PID 2>/dev/null; then
+    echo "ERROR: vLLM server exited early"
+    tail -200 "$LOG"
+    exit 1
+  fi
+  echo "Waiting... ($i/240)"
+  sleep 5
+done
+
+echo "ERROR: vLLM server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1