diff --git a/README.md b/README.md index 97f8400..2ce958d 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ | DSV4 H200 vLLM baseline | `experiments/dsv4_h200_vllm/run_bench.sh` | NVIDIA H200 + vLLM + DeepSeek-V4-Flash | | DSV4 H200 SGLang baseline | `experiments/dsv4_h200_sglang/run_bench.sh` | NVIDIA H200 + native SGLang + DeepSeek-V4-Flash | | DSV4 H200 SGLang vs vLLM | `experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh` | NVIDIA H200 上 SGLang 与 vLLM 控制变量对比（TP=8，最长 200k 上下文） | +| DSV4 H200 vLLM DSpark vs default | `experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh` | vLLM 开启 DSpark 投机解码 vs 默认配置，验证 TTFT 差异 | ### 旧结构（scripts/ + bench_results/） @@ -69,6 +70,12 @@ bash experiments/dsv4_h200_sglang/run_bench.sh bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh ``` +### H200 vLLM DSpark vs default + +```bash +bash experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh +``` + ### DSpark grid（旧结构） ```bash diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/sglang/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/sglang/results.json new file mode 100644 index 0000000..dc5dadb --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/sglang/results.json @@ -0,0 +1,26 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_sglang", + "run_id": "20260708-075719", + "timestamp": "2026-07-08T07:57:23+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "d40e81e", + "git_dirty": "dirty", + "description": "H200 sglang TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "sglang" + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/vllm/results.json b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/vllm/results.json new file mode 100644 index 0000000..c8bc617 --- /dev/null +++ b/experiments/dsv4_h200_sglang_vs_vllm/results/20260708-075719/vllm/results.json @@ -0,0 +1,26 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang_vs_vllm_vllm", + "run_id": "20260708-075719", + "timestamp": "2026-07-08T07:57:23+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "d40e81e", + "git_dirty": "dirty", + "description": "H200 vllm TP=8 comparison benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "phase1_max_model_len": 32768, + "phase2_max_model_len": 210000, + "backend": "vllm" + }, + "scenarios": [] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/README.md b/experiments/dsv4_h200_vllm_dspark_vs_default/README.md new file mode 100644 index 0000000..4dcb34a --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/README.md @@ -0,0 +1,70 @@ +# DSV4 H200 vLLM+DSpark vs vLLM default 对比实验 + +在 8x NVIDIA H200 上，用统一控制变量对比 **vLLM 开启 DSpark 投机解码** 与 **vLLM 默认配置** 的 TTFT / TPOT / 吞吐表现。 + +## 目的 + +你之前的观察是：开启 DSpark 后 TTFT 比默认 vLLM 更长。这个实验把两个配置放在同一套场景下重测，确认现象并量化差异，为后续分析根因提供数据。 + +## 控制变量 + +| 维度 | DSpark | Default | +|---|---|---| +| 硬件 | 8x H200，TP=8，全部 8 张卡 | 同上 | +| 后端 | `vllm serve --spec-method dspark --spec-tokens 5` | 普通 `vllm serve` | +| 压测客户端 | `sglang.bench_serving --backend vllm` | 同上 | +| 场景矩阵 | 5 个场景完全一致 | 同上 | +| KV cache | fp8 | fp8 | +| max_model_len | 32768 | 32768 | +| max_num_seqs | 256 | 256 | + +> 注：DSpark 使用独立的 `DeepSeek-V4-Flash-DSpark` checkpoint；default 使用标准的 `DeepSeek-V4-Flash`。这是 DSpark 方案本身带来的差异， unavoidable。 + +## 场景矩阵（全部测 5 个） + +| Concurrency | Input len | Output len | Num prompts | +|---:|---:|---:|---:| +| 1 | 512 | 256 | 32 | +| 32 | 512 | 256 | 128 | +| 128 | 512 | 256 | 128 | +| 1 | 4000 | 512 | 32 | +| 32 | 4000 | 512 | 64 | + +## 快速运行 + +```bash +bash experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh +``` + +结果保存在 `experiments/dsv4_h200_vllm_dspark_vs_default/results//`： + +``` +results// +├── dspark/ +│ ├── results.json +│ ├── report.md +│ └── raw_outputs/ # gitignored +├── default/ +│ ├── results.json +│ ├── report.md +│ └── raw_outputs/ # gitignored +├── comparison.md +└── logs/ +``` + +## 文件说明 + +| 文件 | 作用 | +|---|---| +| `config.env` | 模型路径、端口、虚拟环境、场景矩阵 | +| `start_dspark.sh` | 启动带 DSpark 投机解码的 vLLM server | +| `start_default.sh` | 启动普通 vLLM server | +| `run_bench.sh` | 总 orchestrator：DSpark → default → 对比报告 | +| `parse_backend.py` | 解析单个 backend 的 raw jsonl 为 `results.json` + `report.md` | +| `compare.py` | 读取两个 backend 结果，生成 TTFT 聚焦的 `comparison.md` | + +## 环境 + +- DSpark server: `/data/user1/yy/envs/vllm-dspark` +- Default server: `/data/user1/yy/envs/vllm` +- Benchmark client: `/data/user1/yy/envs/sglang` diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/compare.py b/experiments/dsv4_h200_vllm_dspark_vs_default/compare.py new file mode 100755 index 0000000..946e682 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/compare.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of vLLM+DSpark and vLLM default. + +Usage: + python3 compare.py --dspark --default \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--dspark", type=Path, required=True) + parser.add_argument("--default", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + args = parser.parse_args() + + dspark_data = load_result(args.dspark) + default_data = load_result(args.default) + + by_scenario = defaultdict(dict) + for data in (dspark_data, default_data): + backend = data["metadata"]["engine"] + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][backend] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write("# vLLM+DSpark vs vLLM default on DeepSeek-V4-Flash (H200, TP=8)\n\n") + f.write("## Summary\n\n") + f.write("- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark`\n") + f.write("- Default model: `/data/models/DeepSeek-V4-Flash`\n") + f.write("- Hardware: 8x NVIDIA H200 143GB\n") + f.write("- Tensor Parallelism: 8\n") + f.write("- Benchmark client: `sglang.bench_serving --backend vllm`\n") + f.write("- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5`\n\n") + + f.write("## Side-by-side results (TTFT focused)\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("vllm-dspark", "vllm-default"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- TTFT mean/P95/P99 are the main focus for verifying whether DSpark increases time-to-first-token.\n") + f.write("- Mean TPOT and E2E are included to check whether speculative decoding pays back after first token.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/config.env b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env new file mode 100644 index 0000000..ce532b5 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/config.env @@ -0,0 +1,47 @@ +# Configuration for the vLLM-dspark vs vLLM-default comparison on H200. +# All values can be overridden via environment variables. + +EXPERIMENT="dsv4_h200_vllm_dspark_vs_default" + +# The dspark variant uses a separate checkpoint and the vllm-dspark env. +DSPARK_MODEL_NAME="DeepSeek-V4-Flash-DSpark" +DSPARK_MODEL_PATH="/data/models/DeepSeek-V4-Flash-DSpark" + +# The default variant uses the standard checkpoint and the plain vllm env. +DEFAULT_MODEL_NAME="DeepSeek-V4-Flash" +DEFAULT_MODEL_PATH="/data/models/DeepSeek-V4-Flash" + +SERVED_MODEL_NAME="deepseek-v4-flash" + +# Different ports so the two backends can be tested independently. +DSPARK_PORT="${DSPARK_PORT:-30007}" +DEFAULT_PORT="${DEFAULT_PORT:-30008}" + +# Virtual environments. +VENV_VLLM_DSPARK="${VENV_VLLM_DSPARK:-/data/user1/yy/envs/vllm-dspark}" +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" + +# Benchmark client always runs from the sglang env for consistency. +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +# Hardware: use all 8 H200 cards for both backends. +export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" +TP=8 + +# Scenarios focused on TTFT: low/high concurrency x short/medium input. +# Each element: "concurrency input_len output_len num_prompts" +if [[ -z "${SCENARIOS:-}" ]]; then + declare -a SCENARIOS=( + "1 512 256 32" + "32 512 256 128" + "128 512 256 128" + "1 4000 512 32" + "32 4000 512 64" + ) +fi +MAX_MODEL_LEN=32768 +MAX_NUM_SEQS=256 + +# Server start scripts bundled with this experiment. +DSPARK_START_SCRIPT="${SCRIPT_DIR:-.}/start_dspark.sh" +DEFAULT_START_SCRIPT="${SCRIPT_DIR:-.}/start_default.sh" diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py b/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py new file mode 100755 index 0000000..cd1f2f1 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/parse_backend.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Parse raw sglang.bench_serving JSONL outputs for one backend. + +Reads JSONL files like {dspark|default}_MMDD_concurrency_inputlen_outputlen.jsonl +and generates results.json + report.md in the given result root. + +Usage: + python3 parse_backend.py [--backend dspark|default] +""" +import argparse +import json +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute_metrics(data: dict) -> dict: + completed = data.get("completed", 0) + total = len(data.get("input_lens", [])) + failed = total - completed if total > 0 else 0 + duration_s = data.get("duration", 0.0) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_token_throughput": data.get("total_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + pretty = "vLLM+DSpark" if backend == "dspark" else "vLLM default" + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# H200 {pretty} Benchmark Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Backend: {pretty} (TP=8)\n") + f.write(f"- Benchmark client: `sglang.bench_serving --backend vllm`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n" + ) + f.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("result_root", type=Path) + parser.add_argument("--backend", default=None, choices=["dspark", "default"]) + args = parser.parse_args() + + result_root = args.result_root + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + backend = args.backend + if backend is None: + for p in raw_dir.iterdir(): + if p.name.startswith("dspark_"): + backend = "dspark" + break + if p.name.startswith("default_"): + backend = "default" + break + if backend is None: + raise SystemExit("Could not infer backend from raw outputs") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + try: + concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) + except ValueError: + continue + + data = parse_jsonl(jsonl_path) + if data is None: + continue + + metrics = compute_metrics(data) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, backend, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh b/experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh new file mode 100755 index 0000000..bcf7cf2 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# vLLM+DSpark vs vLLM default controlled comparison on H200. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "dspark_model=${DSPARK_MODEL_PATH}" +log "default_model=${DEFAULT_MODEL_PATH}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local backend="$1" + local pid_file="/data/user1/yy/${EXPERIMENT_NAME}_${backend}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping ${backend} server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + sleep 2 +} + +start_server() { + local backend="$1" + local start_script port + if [[ "$backend" == "dspark" ]]; then + start_script="${DSPARK_START_SCRIPT}" + port="$DSPARK_PORT" + else + start_script="${DEFAULT_START_SCRIPT}" + port="$DEFAULT_PORT" + fi + + log "starting ${backend} server with ${start_script}" + bash "${start_script}" >> "${log_dir_global}/${backend}.server.outer.log" 2>&1 + + if ! is_server_healthy "$port"; then + log "error: ${backend} server failed to become healthy" + return 1 + fi + log "${backend} server is healthy on port ${port}" +} + +run_warmup() { + local backend="$1" + local port + if [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" + else + port="$DEFAULT_PORT" + fi + + log "warming up ${backend} (input=4000, output=512, num=2)" + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name random \ + --random-input-len 4000 \ + --random-output-len 512 \ + --num-prompts 2 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /tmp/${EXPERIMENT_NAME}_${backend}_warmup.jsonl \ + --output-details \ + > "${log_dir_global}/${backend}.warmup.log" 2>&1 || true + log "warmup for ${backend} completed" +} + +run_benchmark() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + local raw_dir="${result_root}/raw_outputs" + local bench_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$bench_log_dir" + + local port + if [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" + else + port="$DEFAULT_PORT" + fi + + log "===== ${backend} BENCHMARK START =====" + + stop_server "$backend" + start_server "$backend" + run_warmup "$backend" + + for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + output_file="${raw_dir}/${backend}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${bench_log_dir}/${backend}_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: ${backend} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished ${backend} scenario: output=${output_file}" + done + + stop_server "$backend" + log "===== ${backend} BENCHMARK DONE =====" +} + +parse_backend() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + log "parsing ${backend} results in ${result_root}" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \ + >> "${result_root}/logs/parse.log" 2>&1 || { + log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" + } +} + +write_backend_metadata() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + ensure_result_root "$result_root" + local meta_json="${result_root}/results.json" + + local model_path env_path engine + if [[ "$backend" == "dspark" ]]; then + model_path="$DSPARK_MODEL_PATH" + env_path="$VENV_VLLM_DSPARK" + engine="vllm-dspark" + else + model_path="$DEFAULT_MODEL_PATH" + env_path="$VENV_VLLM" + engine="vllm-default" + fi + + write_metadata_json \ + "$meta_json" \ + "${EXPERIMENT_NAME}_${backend}" \ + "$RUN_ID" \ + "$model_path" \ + "vllm" \ + "$engine" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$env_path" \ + "H200 ${engine} TP=8 benchmark for TTFT comparison" + + # Embed config. + "${VENV_CLIENT}/bin/python" - "$meta_json" "$backend" <<'PY' +import json +import sys + +path, backend = sys.argv[1], sys.argv[2] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "max_model_len": 32768, + "max_num_seqs": 256, + "backend": backend, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup. +stop_server dspark +stop_server default + +write_backend_metadata dspark +write_backend_metadata default + +# Run DSpark. +run_benchmark dspark +parse_backend dspark + +# Run default vLLM. +run_benchmark default +parse_backend default + +# Generate comparison. +log "generating comparison report" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ + --dspark "${RESULT_BASE}/${RUN_ID}/dspark" \ + --default "${RESULT_BASE}/${RUN_ID}/default" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/start_default.sh b/experiments/dsv4_h200_vllm_dspark_vs_default/start_default.sh new file mode 100755 index 0000000..89105bf --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/start_default.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Start plain vLLM (no DSpark) for DeepSeek-V4-Flash. +set -e + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/dsv4_h200_vllm_dspark_vs_default_default_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_vllm_dspark_vs_default_default.pid" + +rm -f "$PID_FILE" + +echo "=== Starting DeepSeek-V4-Flash vLLM default baseline (TP=$TP) ===" +echo "Model: $DEFAULT_MODEL_PATH" +echo "Port: $DEFAULT_PORT" +echo "Log: $LOG" + +nohup vllm serve "$DEFAULT_MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$DEFAULT_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${DEFAULT_PORT}/health" >/dev/null 2>&1; then + echo "Default vLLM server is ready at http://127.0.0.1:${DEFAULT_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Default vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Default vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_vllm_dspark_vs_default/start_dspark.sh b/experiments/dsv4_h200_vllm_dspark_vs_default/start_dspark.sh new file mode 100755 index 0000000..d531950 --- /dev/null +++ b/experiments/dsv4_h200_vllm_dspark_vs_default/start_dspark.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Start vLLM with DSpark speculative decoding for DeepSeek-V4-Flash-DSpark. +set -e + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +VENV="${VENV_VLLM_DSPARK}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/dsv4_h200_vllm_dspark_vs_default_dspark_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_vllm_dspark_vs_default_dspark.pid" + +rm -f "$PID_FILE" + +echo "=== Starting DeepSeek-V4-Flash-DSpark with DSpark (TP=$TP) ===" +echo "Model: $DSPARK_MODEL_PATH" +echo "Port: $DSPARK_PORT" +echo "Log: $LOG" + +nohup vllm serve "$DSPARK_MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --spec-method dspark \ + --spec-model "$DSPARK_MODEL_PATH" \ + --spec-tokens 5 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$DSPARK_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${DSPARK_PORT}/health" >/dev/null 2>&1; then + echo "DSpark server is ready at http://127.0.0.1:${DSPARK_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: DSpark server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: DSpark server not healthy after 240 retries" +tail -200 "$LOG" +exit 1