diff --git a/README.md b/README.md index 84fb8b4..97f8400 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ | DSV4 H200 DSpark | `experiments/dsv4_h200_dspark/run_bench.sh` | NVIDIA H200 + vllm-dspark + DeepSeek-V4-Flash-DSpark | | DSV4 H200 vLLM baseline | `experiments/dsv4_h200_vllm/run_bench.sh` | NVIDIA H200 + vLLM + DeepSeek-V4-Flash | | DSV4 H200 SGLang baseline | `experiments/dsv4_h200_sglang/run_bench.sh` | NVIDIA H200 + native SGLang + DeepSeek-V4-Flash | +| DSV4 H200 SGLang vs vLLM | `experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh` | NVIDIA H200 上 SGLang 与 vLLM 控制变量对比（TP=8，最长 200k 上下文） | ### 旧结构（scripts/ + bench_results/） @@ -62,6 +63,12 @@ bash experiments/dsv4_h200_dspark/run_bench.sh bash experiments/dsv4_h200_sglang/run_bench.sh ``` +### H200 SGLang vs vLLM 对比 + +```bash +bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh +``` + ### DSpark grid（旧结构） ```bash diff --git a/experiments/dsv4_h200_sglang_vs_vllm/README.md b/experiments/dsv4_h200_sglang_vs_vllm/README.md new file mode 100644 index 0000000..0b5b729 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/README.md @@ -0,0 +1,72 @@ +# DSV4 H200 SGLang vs vLLM 对比实验 + +在 8x NVIDIA H200 上，用统一控制变量对比 SGLang 与 vLLM serving DeepSeek-V4-Flash。 + +## 设计原则 + +- 两个 backend 都使用 **TP=8**，绑定全部 8 张卡。 +- 压测客户端统一使用 `sglang.bench_serving`。 +- 场景矩阵、并发、输入/输出长度、请求数对两个 backend 完全一致。 +- 每个 phase 启动 server 后先做 warmup，再跑正式压测。 +- 短上下文和长上下文分阶段启动 server，避免 `max-model-len` / `max-num-seqs` 显存冲突。 + +## 场景矩阵 + +### Phase 1：短上下文吞吐（max_model_len=32k） + +| Concurrency | Input | Output | Num prompts | +|---:|---:|---:|---:| +| 1 | 512 | 256 | 32 | +| 32 | 512 | 256 | 128 | +| 128 | 512 | 256 | 128 | +| 1 | 4000 | 512 | 32 | +| 32 | 4000 | 512 | 64 | + +### Phase 2：长上下文（max_model_len=210k） + +| Concurrency | Input | Output | Num prompts | +|---:|---:|---:|---:| +| 1 | 32768 | 1024 | 8 | +| 1 | 65536 | 1024 | 4 | +| 1 | 131072 | 1024 | 2 | +| 1 | 200000 | 1024 | 1 | + +## 快速运行 + +```bash +bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh +``` + +结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results//`： + +``` +results// +├── sglang/ +│ ├── results.json +│ ├── report.md +│ └── raw_outputs/ # gitignored +├── vllm/ +│ ├── results.json +│ ├── report.md +│ └── raw_outputs/ # gitignored +├── comparison.md +└── logs/ +``` + +## 文件说明 + +| 文件 | 作用 | +|---|---| +| `config.env` | 公共配置：模型路径、端口、场景矩阵、虚拟环境 | +| `start_sglang.sh` | 按 phase 启动 SGLang server | +| `start_vllm.sh` | 按 phase 启动 vLLM server | +| `run_bench.sh` | 总 orchestrator：跑 SGLang → 跑 vLLM → 生成对比报告 | +| `warmup.py` | 用 `sglang.bench_serving` 发送少量预热请求 | +| `parse_backend.py` | 解析单个 backend 的 raw jsonl 为 `results.json` + `report.md` | +| `compare.py` | 读取两个 backend 的 `results.json`，生成 `comparison.md` | + +## 环境 + +- SGLM server: `/data/user1/yy/envs/sglang` +- vLLM server: `/data/user1/yy/envs/vllm` +- Benchmark client: `/data/user1/yy/envs/sglang`（统一用该环境的 `sglang.bench_serving`） diff --git a/experiments/dsv4_h200_sglang_vs_vllm/compare.py b/experiments/dsv4_h200_sglang_vs_vllm/compare.py new file mode 100755 index 0000000..0f4eb51 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/compare.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of SGLang and vLLM results. + +Usage: + python3 compare.py --sglang --vllm \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str: + # S2 tier targets from scripts/SLO_STANDARDS.md. + ttft_ok = ttft_p95_ms < 3000.0 + tpot_ok = tpot_mean_ms < 50.0 + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sglang", type=Path, required=True) + parser.add_argument("--vllm", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + args = parser.parse_args() + + sglang_data = load_result(args.sglang) + vllm_data = load_result(args.vllm) + + by_scenario = defaultdict(dict) + for data in (sglang_data, vllm_data): + backend = data["metadata"]["engine"] + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][backend] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n") + f.write("## Summary\n\n") + f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") + f.write("- Hardware: 8x NVIDIA H200 143GB\n") + f.write("- Tensor Parallelism: 8\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n") + + f.write("## Side-by-side results\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("sglang", "vllm"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"]) + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_sglang_vs_vllm/config.env b/experiments/dsv4_h200_sglang_vs_vllm/config.env new file mode 100644 index 0000000..ca5e2d8 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/config.env @@ -0,0 +1,55 @@ +# Common configuration for the SGLang vs vLLM comparison on H200. +# All values can be overridden via environment variables. + +EXPERIMENT="dsv4_h200_sglang_vs_vllm" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Ports must differ so both backends can be tested independently. +SGLANG_PORT="${SGLANG_PORT:-30006}" +VLLM_PORT="${VLLM_PORT:-30005}" + +# Virtual environments. +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" + +# Hardware: use all 8 H200 cards for both backends. +export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" +TP=8 + +# Phase 1: short-context throughput (max_model_len 32k). +# Each element: "concurrency input_len output_len num_prompts" +if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then + declare -a PHASE1_SCENARIOS=( + "1 512 256 32" + "32 512 256 128" + "128 512 256 128" + "1 4000 512 32" + "32 4000 512 64" + ) +fi +PHASE1_MAX_MODEL_LEN=32768 +PHASE1_MAX_NUM_SEQS=256 +PHASE1_MAX_RUNNING=256 + +# Phase 2: long context (max_model_len 210k to leave room for output + padding). +if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then + declare -a PHASE2_SCENARIOS=( + "1 32768 1024 8" + "1 65536 1024 4" + "1 131072 1024 2" + "1 200000 1024 1" + ) +fi +PHASE2_MAX_MODEL_LEN=210000 +PHASE2_MAX_NUM_SEQS=2 +PHASE2_MAX_RUNNING=2 + +# Benchmark client always runs from the sglang env so that the same +# sglang.bench_serving version is used for both backends. +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" + +# Server start scripts bundled with this experiment. +SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" +VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" diff --git a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py new file mode 100755 index 0000000..9da7392 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Parse raw sglang.bench_serving JSONL outputs for one backend. + +Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl +and generates results.json + report.md in the given result root. + +Usage: + python3 parse_backend.py [--backend sglang|vllm] +""" +import argparse +import json +import re +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute_metrics(data: dict) -> dict: + completed = data.get("completed", 0) + total = len(data.get("input_lens", [])) + failed = total - completed if total > 0 else 0 + duration_s = data.get("duration", 0.0) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_token_throughput": data.get("total_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n") + f.write(f"- Backend: {backend.upper()} (TP=8)\n") + f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + f.write( + f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n" + ) + f.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("result_root", type=Path) + parser.add_argument("--backend", default=None, choices=["sglang", "vllm"]) + args = parser.parse_args() + + result_root = args.result_root + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + backend = args.backend + if backend is None: + # Infer from filenames if not provided. + for p in raw_dir.iterdir(): + if p.name.startswith("sglang_"): + backend = "sglang" + break + if p.name.startswith("vllm_"): + backend = "vllm" + break + if backend is None: + raise SystemExit("Could not infer backend from raw outputs") + + # Filename: {backend}_phase{1|2}_MMDD_concurrency_inputlen_outputlen.jsonl + pattern = re.compile(rf"^{backend}_phase(\d+)_(\d+)_(\d+)_(\d+)_(\d+)\.jsonl$") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob(f"{backend}_phase*.jsonl")): + m = pattern.match(jsonl_path.name) + if not m: + continue + phase_num, concurrency, input_len, output_len = m.groups() + concurrency, input_len, output_len = int(concurrency), int(input_len), int(output_len) + + data = parse_jsonl(jsonl_path) + if data is None: + continue + + metrics = compute_metrics(data) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "phase": f"phase{phase_num}", + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, backend, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh new file mode 100755 index 0000000..1f3e450 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# SGLang vs vLLM controlled comparison on H200. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local backend="$1" + local pid_file="/data/user1/yy/dsv4_h200_sglang_vs_vllm_${backend}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping ${backend} server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + # Fallback cleanup. + if [[ "$backend" == "sglang" ]]; then + pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true + else + pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true + fi + sleep 2 +} + +start_server() { + local backend="$1" + local phase="$2" + local start_script + if [[ "$backend" == "sglang" ]]; then + start_script="${SGLANG_START_SCRIPT}" + local port="$SGLANG_PORT" + else + start_script="${VLLM_START_SCRIPT}" + local port="$VLLM_PORT" + fi + + log "starting ${backend} server (phase=${phase}) with ${start_script}" + bash "${start_script}" "$phase" >> "${log_dir_global}/${backend}_${phase}.server.outer.log" 2>&1 + + if ! is_server_healthy "$port"; then + log "error: ${backend} server (phase=${phase}) failed to become healthy" + return 1 + fi + log "${backend} server is healthy on port ${port}" +} + +run_warmup() { + local backend="$1" + local phase="$2" + local port input_len output_len num + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + if [[ "$phase" == "phase1" ]]; then + input_len=4000 + output_len=512 + num=2 + else + input_len=200000 + output_len=1024 + num=1 + fi + + log "warming up ${backend} (phase=${phase}, input=${input_len}, output=${output_len}, num=${num})" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \ + --backend "$backend" \ + --port "$port" \ + --input-len "$input_len" \ + --output-len "$output_len" \ + --num "$num" \ + --env-python "${VENV_CLIENT}/bin/python" \ + >> "${log_dir_global}/${backend}_${phase}.warmup.log" 2>&1 + log "warmup for ${backend} ${phase} completed" +} + +run_phase() { + local backend="$1" + local phase="$2" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + local raw_dir="${result_root}/raw_outputs" + local phase_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$phase_log_dir" + + local -n scenarios + local max_model_len + if [[ "$phase" == "phase1" ]]; then + scenarios=PHASE1_SCENARIOS + max_model_len="$PHASE1_MAX_MODEL_LEN" + else + scenarios=PHASE2_SCENARIOS + max_model_len="$PHASE2_MAX_MODEL_LEN" + fi + + local port + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + log "===== ${backend} ${phase} START (max_model_len=${max_model_len}) =====" + + stop_server "$backend" + start_server "$backend" "$phase" + run_warmup "$backend" "$phase" + + for scenario in "${scenarios[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend "$backend" \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: ${backend} ${phase} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished ${backend} ${phase} scenario: output=${output_file}" + done + + stop_server "$backend" + log "===== ${backend} ${phase} DONE =====" +} + +parse_backend() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + log "parsing ${backend} results in ${result_root}" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \ + >> "${result_root}/logs/parse.log" 2>&1 || { + log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" + } +} + +write_backend_metadata() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + ensure_result_root "$result_root" + local meta_json="${result_root}/results.json" + + local env_path + if [[ "$backend" == "sglang" ]]; then + env_path="$VENV_SGLANG" + else + env_path="$VENV_VLLM" + fi + + write_metadata_json \ + "$meta_json" \ + "${EXPERIMENT_NAME}_${backend}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$backend" \ + "$backend" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$env_path" \ + "H200 ${backend} TP=8 comparison benchmark for DeepSeek-V4-Flash" + + # Embed config. + "${VENV_CLIENT}/bin/python" - "$meta_json" "$backend" <<'PY' +import json +import sys + +path, backend = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": backend, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +stop_server sglang +stop_server vllm + +write_backend_metadata sglang +write_backend_metadata vllm + +# Run SGLang. +run_phase sglang phase1 +run_phase sglang phase2 +parse_backend sglang + +# Run vLLM. +run_phase vllm phase1 +run_phase vllm phase2 +parse_backend vllm + +# Generate comparison. +log "generating comparison report" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ + --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ + --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_sglang_vs_vllm/start_sglang.sh b/experiments/dsv4_h200_sglang_vs_vllm/start_sglang.sh new file mode 100755 index 0000000..0bc7777 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/start_sglang.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Start SGLang server for the comparison, in either phase1 or phase2 mode. +set -e + +PHASE="${1:-phase1}" +if [[ "$PHASE" != "phase1" && "$PHASE" != "phase2" ]]; then + echo "Usage: $0 {phase1|phase2}" + exit 1 +fi + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +export PATH="${VENV_SGLANG}/bin:$PATH" +export PYTHONUNBUFFERED=1 +export SGLANG_LOG_LEVEL=info +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +if [[ "$PHASE" == "phase1" ]]; then + MAX_MODEL_LEN="$PHASE1_MAX_MODEL_LEN" + MAX_RUNNING="$PHASE1_MAX_RUNNING" +else + MAX_MODEL_LEN="$PHASE2_MAX_MODEL_LEN" + MAX_RUNNING="$PHASE2_MAX_RUNNING" +fi + +LOG="/data/user1/yy/logs/dsv4_h200_sglang_vs_vllm_sglang_${PHASE}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_sglang_vs_vllm_sglang.pid" + +rm -f "$PID_FILE" + +echo "=== Starting SGLang server (TP=$TP, phase=$PHASE, max_model_len=$MAX_MODEL_LEN) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Log: $LOG" + +nohup sglang serve \ + --trust-remote-code \ + --model-path "$MODEL_PATH" \ + --tp "$TP" \ + --moe-runner-backend marlin \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-running-requests "$MAX_RUNNING" \ + --mem-fraction-static 0.88 \ + --host 0.0.0.0 \ + --port "$SGLANG_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_sglang_vs_vllm/start_vllm.sh b/experiments/dsv4_h200_sglang_vs_vllm/start_vllm.sh new file mode 100755 index 0000000..1ad085b --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/start_vllm.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Start vLLM server for the comparison, in either phase1 or phase2 mode. +set -e + +PHASE="${1:-phase1}" +if [[ "$PHASE" != "phase1" && "$PHASE" != "phase2" ]]; then + echo "Usage: $0 {phase1|phase2}" + exit 1 +fi + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +if [[ "$PHASE" == "phase1" ]]; then + MAX_MODEL_LEN="$PHASE1_MAX_MODEL_LEN" + MAX_NUM_SEQS="$PHASE2_MAX_NUM_SEQS" +else + MAX_MODEL_LEN="$PHASE2_MAX_MODEL_LEN" + MAX_NUM_SEQS="$PHASE2_MAX_NUM_SEQS" +fi + +LOG="/data/user1/yy/logs/dsv4_h200_sglang_vs_vllm_vllm_${PHASE}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_sglang_vs_vllm_vllm.pid" + +rm -f "$PID_FILE" + +echo "=== Starting vLLM server (TP=$TP, phase=$PHASE, max_model_len=$MAX_MODEL_LEN, max_num_seqs=$MAX_NUM_SEQS) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Log: $LOG" + +nohup vllm serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$VLLM_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_sglang_vs_vllm/warmup.py b/experiments/dsv4_h200_sglang_vs_vllm/warmup.py new file mode 100755 index 0000000..75dbe5f --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/warmup.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Send a small number of warmup requests to a running backend. + +Uses sglang.bench_serving with a single request so that the same code path +(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real +benchmark begins. Discards the output. +""" +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + + +def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None: + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp: + cmd = [ + str(env_python), + "-m", + "sglang.bench_serving", + "--backend", + backend, + "--host", + host, + "--port", + str(port), + "--dataset-name", + "random", + "--random-input-len", + str(input_len), + "--random-output-len", + str(output_len), + "--num-prompts", + str(num), + "--max-concurrency", + "1", + "--request-rate", + "10000", + "--output-file", + tmp.name, + "--output-details", + ] + print(f"[warmup] {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("[warmup] FAILED", file=sys.stderr) + print(result.stdout, file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}") + + +def main(): + parser = argparse.ArgumentParser(description="Warmup a serving backend.") + parser.add_argument("--backend", required=True, choices=["sglang", "vllm"]) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--input-len", type=int, required=True) + parser.add_argument("--output-len", type=int, required=True) + parser.add_argument("--num", type=int, default=1) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python") + args = parser.parse_args() + + run_warmup( + backend=args.backend, + host=args.host, + port=args.port, + input_len=args.input_len, + output_len=args.output_len, + num=args.num, + env_python=Path(args.env_python), + ) + + +if __name__ == "__main__": + main()