From 04a384b265419349a5ae9f8db4620246ac200ebe Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Thu, 9 Jul 2026 09:55:18 +0000 Subject: [PATCH] =?UTF-8?q?add=20dsv4=5Fh200=5Fvllm=5Ftp=5Fdp=5Fmatrix=20e?= =?UTF-8?q?xperiment=20(TP=C3=97DP=20matrix=20with=20Y-only=20mode,=201M?= =?UTF-8?q?=20excluded)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dsv4_h200_vllm_tp_dp_matrix/compare.py | 162 +++++++ .../dsv4_h200_vllm_tp_dp_matrix/config.env | 43 ++ .../generate_scenarios.py | 105 +++++ .../dsv4_h200_vllm_tp_dp_matrix/matrix.json | 81 ++++ .../dsv4_h200_vllm_tp_dp_matrix/run_bench.sh | 394 ++++++++++++++++++ .../smoke_matrix.json | 18 + .../start_vllm_dp.sh | 92 ++++ scripts/common/parse_backend.py | 97 ++++- 8 files changed, 987 insertions(+), 5 deletions(-) create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/config.env create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh create mode 100644 experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json create mode 100755 experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py new file mode 100755 index 0000000..8bf26df --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h200_vllm_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: vLLM\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, x.replace("c", "").replace("i", "_").replace("o", "_").split("_")[1:]))): + cfg_part, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + # cfg_part not used; just for readability. + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/DSL. + f.write("\n## Best throughput per (ISL, DSL)\n\n") + f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split() + isl_i, dsl_i = int(isl), int(dsl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]: + best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s) + for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env new file mode 100644 index 0000000..b9f0398 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/config.env @@ -0,0 +1,43 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs). +# Tests vLLM with three parallel configurations: +# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +VLLM_PORT="${VLLM_PORT:-30030}" +# vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port. +VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}" + +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +# The benchmark client is sglang.bench_serving, even when the backend is vLLM. +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# Matrix and concurrency rules are defined in matrix.json. +# This file is consumed by generate_scenarios.py. +MATRIX_FILE="${SCRIPT_DIR:-.}/matrix.json" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. 0 means use the default heuristic in +# generate_scenarios.py (6-8 points, or all integers when range is small). +CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py new file mode 100755 index 0000000..ef46e01 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return a sorted list of concurrency values in [low, high]. + + If the range is small, return every integer. Otherwise sample roughly + `target` points linearly between low and high (inclusive). + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if high - low + 1 <= target: + return list(range(low, high + 1)) + + points = set() + points.add(low) + points.add(high) + step = (high - low) / (target - 1) + for i in range(1, target - 1): + v = low + round(step * i) + points.add(max(low, min(high, v))) + return sorted(points) + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + dsl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for dsl_str in sorted(dsl_map.keys(), key=int): + mark = dsl_map[dsl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(dsl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..1ed8e9f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json @@ -0,0 +1,81 @@ +{ + "comment": "ISL/DSL matrix for dsv4_h200_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.", + "mode": "Y", + "comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + }, + "262144": { + "128": "Y", + "256": "Y", + "512": "P", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "524288": { + "128": "Y", + "256": "P", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + }, + "1048576": { + "128": "N", + "256": "N", + "512": "N", + "1024": "N", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 }, + "262144": { "low": 1, "high": 2 }, + "524288": { "low": 1, "high": 2 }, + "1048576": { "low": 1, "high": 2 } + } +} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..0df25b4 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh @@ -0,0 +1,394 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y+P}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +PYTHON="${VENV_CLIENT}/bin/python" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +health_port_for() { + local dp="$1" + if [[ "$dp" -gt 1 ]]; then + echo "$VLLM_DP_SUPERVISOR_PORT" + else + echo "$VLLM_PORT" + fi +} + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + + # Fallback: kill any vllm serve processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + local max_model_len="$3" + local max_num_seqs="$4" + + local args=( + "vllm serve" "$MODEL_PATH" + --trust-remote-code + --tensor-parallel-size "$tp" + --kv-cache-dtype fp8 + --max-model-len "$max_model_len" + --max-num-seqs "$max_num_seqs" + --block-size 256 + --gpu-memory-utilization 0.90 + --tokenizer-mode deepseek_v4 + --reasoning-parser deepseek_v4 + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --port "$VLLM_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --data-parallel-size "$dp" + --data-parallel-size-local "$dp" + --data-parallel-multi-port-external-lb + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + local max_model_len="$3" + local max_num_seqs="$4" + + log "starting vllm server tp=${tp} dp=${dp} max_model_len=${max_model_len} max_num_seqs=${max_num_seqs}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" "$max_model_len" "$max_num_seqs" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + local hport + hport="$(health_port_for "$dp")" + if ! is_server_healthy "$hport"; then + log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}" + return 1 + fi + log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}" +} + +run_warmup() { + local tp="$1" + local dp="$2" + local input_len="$3" + local output_len="$4" + local hport + hport="$(health_port_for "$dp")" + + log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$hport" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num 1 \ + --env-python "$PYTHON" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1 + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + # Remaining args are key=value pairs. + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {k: json.loads(v) for k, v in pairs} +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + if [[ "$total_scenarios" -eq 0 ]]; then + log "no scenarios for ${config_label}; skipping" + return 0 + fi + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "vllm" \ + "vllm" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_VLLM" \ + "H200 vLLM TP×DP matrix for DeepSeek-V4-Flash" + + # Embed static config now; server_args will be updated per ISL group. + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "vllm", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh" + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + # Group scenarios by input_len. We start one server per input_len with a + # max_model_len large enough for the biggest output_len in that group. + local current_isl="" + local max_dsl_for_isl=0 + local max_conc_for_isl=0 + local group_started=false + + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do + # When the input length changes, restart the server for the new group. + if [[ "$isl" != "$current_isl" ]]; then + if [[ "$group_started" == true ]]; then + stop_server "$tp" "$dp" + fi + current_isl="$isl" + max_dsl_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($3>max) max=$3} END{print max+0}' "$scenario_tsv")" + max_conc_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($4>max) max=$4} END{print max+0}' "$scenario_tsv")" + + local max_model_len=$((isl + max_dsl_for_isl + 64)) + local max_num_seqs=$((max_conc_for_isl + 8)) + + stop_server "$tp" "$dp" + if ! start_server "$tp" "$dp" "$max_model_len" "$max_num_seqs"; then + log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group" + group_started=false + continue + fi + group_started=true + + # Update metadata with the actual server args for this ISL group. + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp" "$max_model_len" "$max_num_seqs")" + jq --arg isl "$isl" --arg args "$server_args_str" \ + '.config.server_args_per_isl += {($isl): $args}' \ + "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + # Warmup with the shortest output for this ISL. + run_warmup "$tp" "$dp" "$isl" 128 + fi + + # If the server for this group did not start, skip all scenarios in it. + if [[ "$group_started" != true ]]; then + log "skipping ${config_label} ISL=${isl} scenario (server not started)" + continue + fi + + local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl" + local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log" + local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv" + + if scenario_already_completed "$output_file" "$num"; then + log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + local bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" \ + "$PYTHON" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$(health_port_for "$dp")" \ + --dataset-name random \ + --random-input-len "$isl" \ + --random-output-len "$dsl" \ + --num-prompts "$num" \ + --max-concurrency "$conc" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -ne 0 ]]; then + log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped_oom and continuing" + append_scenario_record "$result_root" \ + "name=c${conc}_i${isl}_o${dsl}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_oom\"" \ + "note=\"bench command failed or timed out (rc=${bench_rc})\"" + continue + else + log "mandatory (Y) scenario failed; aborting current ISL group" + stop_server "$tp" "$dp" + group_started=false + continue + fi + fi + + log "finished ${config_label} scenario: output=${output_file}" + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json b/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json new file mode 100644 index 0000000..5715c28 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json @@ -0,0 +1,18 @@ +{ + "comment": "Small smoke matrix for quickly verifying TP×DP server startup and bench path.", + "mode": "Y", + "matrix": { + "1024": { + "128": "Y", + "256": "Y" + }, + "4096": { + "128": "Y", + "256": "Y" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 8 }, + "4096": { "low": 1, "high": 4 } + } +} diff --git a/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh new file mode 100755 index 0000000..2ae2fee --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +set -e + +TP="${1}" +DP="${2}" +MAX_MODEL_LEN="${3}" +MAX_NUM_SEQS="${4}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +# Build the base command as an array. +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --tensor-parallel-size "$TP" + --kv-cache-dtype fp8 + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --block-size 256 + --gpu-memory-utilization 0.90 + --tokenizer-mode deepseek_v4 + --reasoning-parser deepseek_v4 + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + --data-parallel-size-local "$DP" + --data-parallel-multi-port-external-lb + ) + # vLLM 0.24.0 hard-codes the DP supervisor port to 9256 in + # entrypoints/openai/cli_args.py. The bench client and health checks must + # target that port when DP > 1. + HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT" +else + HEALTH_PORT="$VLLM_PORT" +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}, max_model_len=${MAX_MODEL_LEN}, max_num_seqs=${MAX_NUM_SEQS}) ===" +echo "Model: $MODEL_PATH" +echo "Health port: $HEALTH_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${HEALTH_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${HEALTH_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${HEALTH_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/scripts/common/parse_backend.py b/scripts/common/parse_backend.py index f07e6c6..d269dc1 100755 --- a/scripts/common/parse_backend.py +++ b/scripts/common/parse_backend.py @@ -93,6 +93,72 @@ def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: floa } +def parse_gpu_memory_csv(jsonl_path: Path) -> dict | None: + """Parse a paired nvidia-smi CSV for GPU memory/utilization statistics. + + The CSV is expected to live in a sibling `gpu_logs/` directory or in the same + `raw_outputs/` directory, named `gpu_mem_c_i_o.csv`. + """ + result_root = jsonl_path.parent.parent + scenario_id = jsonl_path.stem.split("_", 2)[2] # e.g. c1_i1024_o128 + csv_name = f"gpu_mem_{scenario_id}.csv" + + csv_path = None + for candidate in ( + result_root / "gpu_logs" / csv_name, + result_root / "raw_outputs" / csv_name, + ): + if candidate.exists() and candidate.stat().st_size > 0: + csv_path = candidate + break + + if csv_path is None: + return None + + per_gpu = {} + total_mb = None + try: + with open(csv_path, "r", encoding="utf-8") as f: + header = f.readline() + if not header.strip(): + return None + for line in f: + line = line.strip() + if not line: + continue + parts = [p.strip() for p in line.split(",")] + if len(parts) < 5: + continue + idx = parts[1] + try: + used = float(parts[2].split()[0]) + total = float(parts[3].split()[0]) + util = float(parts[4].split()[0]) + except (ValueError, IndexError): + continue + per_gpu.setdefault(idx, {"used": [], "util": []}) + per_gpu[idx]["used"].append(used) + per_gpu[idx]["util"].append(util) + if total_mb is None: + total_mb = total + except Exception: + return None + + if not per_gpu or total_mb is None: + return None + + peak_used = max(max(g["used"]) for g in per_gpu.values()) + avg_used = sum(sum(g["used"]) / len(g["used"]) for g in per_gpu.values()) / len(per_gpu) + peak_util = max(max(g["util"]) for g in per_gpu.values()) + + return { + "peak_used_mb": peak_used, + "avg_used_mb": avg_used, + "peak_utilization_pct": peak_util, + "memory_total_mb": total_mb, + } + + def generate_report(result_root: Path, backend: str, scenarios: list[dict], metadata: dict | None) -> None: report_path = result_root / "report.md" model = metadata.get("model", "unknown") if metadata else "unknown" @@ -106,13 +172,18 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") f.write("## Results\n\n") - f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") for s in scenarios: cfg = s["config"] m = s["metrics"] slo = s.get("slo_status", {}).get("overall", "") + gpu = m.get("gpu_memory") + if gpu: + gpu_str = f"{gpu['peak_used_mb']:.0f}/{gpu['memory_total_mb']:.0f} MiB ({100*gpu['peak_used_mb']/gpu['memory_total_mb']:.1f}%)" + else: + gpu_str = "-" f.write( f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " @@ -120,7 +191,7 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta f"{m['total_token_throughput']:.2f} | " f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {gpu_str} | {slo} |\n" ) f.write("\n") f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") @@ -152,10 +223,13 @@ def main() -> None: raise SystemExit("Could not infer backend from raw outputs") metadata = None + old_scenarios = [] if results_json.exists(): with open(results_json, "r", encoding="utf-8") as f: try: - metadata = json.load(f).get("metadata") + existing = json.load(f) + metadata = existing.get("metadata") + old_scenarios = existing.get("scenarios", []) except json.JSONDecodeError: pass @@ -183,6 +257,7 @@ def main() -> None: continue metrics = compute_metrics(data) + metrics["gpu_memory"] = parse_gpu_memory_csv(jsonl_path) scenario = { "name": scenario_name(concurrency, input_len, output_len), "config": { @@ -199,10 +274,22 @@ def main() -> None: } scenarios.append(scenario) - if not scenarios: + if not scenarios and not old_scenarios: print("No benchmark outputs found to parse") return + # Preserve any manually-recorded skipped/failed scenarios (e.g. optional + # combinations that OOMed) that do not have a fresh raw output. + parsed_names = {s["name"] for s in scenarios} + for old in old_scenarios: + if old.get("status") and old["name"] not in parsed_names: + scenarios.append(old) + scenarios.sort(key=lambda s: ( + s["config"]["input_len"], + s["config"]["output_len"], + s["config"]["concurrency"], + )) + if results_json.exists(): with open(results_json, "r", encoding="utf-8") as f: data = json.load(f)