From 9b485c05f7a65ea0ab196230c768a0c90da8a81b Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 11:04:16 +0000 Subject: [PATCH] cleanup(scripts): remove unused legacy scripts; update README and workflow docs --- BENCHMARK_WORKFLOW.md | 25 +- README.md | 23 +- docs/H200_QUICKSTART.md | 8 +- .../20260708-100016/sglang/results.json | 42 +++ .../results/20260708-100016/vllm/results.json | 42 +++ scripts/analysis/compare_experiments.py | 77 ----- scripts/analyze_dspark_nsys.py | 87 ----- scripts/bench_dsv4_comparison.py | 323 ------------------ scripts/bench_vllm_dspark_qwen3.py | 266 --------------- scripts/benchmark_dsv4_backend_comparison.sh | 128 ------- scripts/install_vllm_dspark.sh | 27 -- scripts/profile_dspark.py | 261 -------------- scripts/run_sglang_benchmark.sh | 157 --------- scripts/run_sglang_max_throughput.sh | 143 -------- scripts/start_dsv4_dspark_8card_bf16kv.sh | 63 ---- scripts/start_dsv4_dspark_8card_flashinfer.sh | 66 ---- scripts/start_dsv4_dspark_8card_param.sh | 85 ----- scripts/start_pd_single_node.sh | 138 -------- scripts/start_sglang_dsv4_8card.sh | 27 -- scripts/start_vllm_eagle_dsv4_8card.sh | 65 ---- scripts/start_vllm_mtp_dsv4_8card.sh | 66 ---- scripts/start_vllm_pd_single_node.sh | 156 --------- scripts/wait_and_profile_dspark.sh | 60 ---- 23 files changed, 99 insertions(+), 2236 deletions(-) create mode 100644 experiments/dsv4_h200_long_context_matrix/results/20260708-100016/sglang/results.json create mode 100644 experiments/dsv4_h200_long_context_matrix/results/20260708-100016/vllm/results.json delete mode 100644 scripts/analysis/compare_experiments.py delete mode 100644 scripts/analyze_dspark_nsys.py delete mode 100644 scripts/bench_dsv4_comparison.py delete mode 100644 scripts/bench_vllm_dspark_qwen3.py delete mode 100644 scripts/benchmark_dsv4_backend_comparison.sh delete mode 100755 scripts/install_vllm_dspark.sh delete mode 100755 scripts/profile_dspark.py delete mode 100755 scripts/run_sglang_benchmark.sh delete mode 100755 scripts/run_sglang_max_throughput.sh delete mode 100755 scripts/start_dsv4_dspark_8card_bf16kv.sh delete mode 100755 scripts/start_dsv4_dspark_8card_flashinfer.sh delete mode 100755 scripts/start_dsv4_dspark_8card_param.sh delete mode 100755 scripts/start_pd_single_node.sh delete mode 100755 scripts/start_sglang_dsv4_8card.sh delete mode 100644 scripts/start_vllm_eagle_dsv4_8card.sh delete mode 100644 scripts/start_vllm_mtp_dsv4_8card.sh delete mode 100755 scripts/start_vllm_pd_single_node.sh delete mode 100755 scripts/wait_and_profile_dspark.sh diff --git a/BENCHMARK_WORKFLOW.md b/BENCHMARK_WORKFLOW.md index 4958067..e36f593 100644 --- a/BENCHMARK_WORKFLOW.md +++ b/BENCHMARK_WORKFLOW.md @@ -9,10 +9,9 @@ │ ├── nvidia_h200.env │ └── patches/kunlun_p800/ # runtime patches required by some images ├── scripts/ # shared benchmark/orchestrator/utility scripts -│ ├── common/ # reusable components (lib.sh, platform.sh, ...) -│ ├── analysis/ # cross-experiment comparison tools -│ ├── benchmark_dspark_0707/ # legacy DSpark benchmark suite -│ └── ... +│ ├── common/ # reusable components (lib.sh, platform.sh, parse_backend.py, ...) +│ ├── benchmark_dspark_0707/ # legacy DSpark benchmark suite (仍被 experiments/dsv4_h200_dspark/ 引用) +│ └── SLO_STANDARDS.md # 推理服务 SLO 标准 ├── experiments/ # experiment-centric directories (preferred) │ └── dsv4_p800_sglang/ │ ├── README.md @@ -38,7 +37,7 @@ 1. **Experiments are the primary organization unit.** - Each experiment lives under `experiments//` and contains its scripts, configuration, and results. - Shared orchestration code lives in `scripts/common/`; do not copy server start / health check logic into every experiment. - - Legacy experiments may remain under `scripts//` with outputs in `bench_results/`, but new work should use `experiments/`. + - 新实验必须放在 `experiments//` 下；`scripts/` 下仅保留 `scripts/common/` 和被现有实验引用的 legacy 脚本。 2. **Benchmark outputs live with their experiment.** - For `experiments//`, results go in `experiments//results//`. @@ -266,13 +265,7 @@ python3 experiments/dsv4_p800_sglang/parse_results.py \ # Legacy layout /data/user1/yy/envs/sglang/bin/python scripts/benchmark_dspark_0707/parse_results.py \ - /data/user1/yy/bench_results/dspark_grid_ -``` - -### Cross-experiment comparison - -```bash -python3 scripts/analysis/compare_experiments.py + /data/user1/yy/experiments/legacy_bench_results/dspark_grid_ ``` ## Adding a New Platform or Experiment @@ -320,11 +313,11 @@ At minimum, `run_bench.sh` should: - `scripts/common/lib.sh`: logging, health checks, metadata JSON. - `scripts/common/platform.sh`: platform auto-detection and env loading. -- `scripts/common/server_docker.sh`: Docker-based server lifecycle. -- `scripts/common/bench_client_docker.sh`: run `sglang.bench_serving` inside a container. +- `scripts/common/warmup.py`: server 预热。 +- `scripts/common/parse_backend.py`: raw jsonl -> `results.json` + `report.md`。 +- `scripts/common/compare.py`: SGLang vs vLLM 横向对比表。 -Native (non-Docker) platforms can delegate to existing start scripts or add -new shared helpers under `scripts/common/`. +Docker/XPU 平台可以在 `scripts/common/` 下新增自己的 helper，但不要在实验目录复制通用逻辑。 ### 4. Example experiments diff --git a/README.md b/README.md index b135e7a..203a7e8 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,8 @@ | 目录/文件 | 说明 | |---|---| | `platforms/` | 芯片/加速器平台配置（`*.env`），如 P800、H200 | -| `scripts/common/` | 跨实验复用的 orchestration 组件（server 启停、health check、元数据生成） | -| `scripts/analysis/` | 跨实验对比分析工具 | -| `scripts/` | 旧版/legacy benchmark 脚本 | +| `scripts/common/` | 跨实验复用的 orchestration 组件（server 启停、health check、元数据生成、结果解析） | +| `scripts/` | 仅保留仍在使用的 legacy DSpark 脚本与 SLO 标准 | | `experiments/` | 以实验为单位的自包含目录（脚本 + 配置 + 结果） | | `experiments/legacy_bench_results/` | 历史归档的实验产物（只读），从原 `bench_results/` 迁移而来 | | `BENCHMARK_WORKFLOW.md` | benchmark 目录与命名规范 | @@ -34,16 +33,12 @@ ### 旧结构（scripts/ + bench_results/） +仅保留还在被 `experiments/dsv4_h200_dspark/` 引用的 legacy DSpark 脚本，其余旧脚本已删除。 + | 实验 | 脚本 | 最终报告 | 说明 | |---|---|---|---| | DSpark grid benchmark | `scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh` | `experiments/legacy_bench_results/dspark_grid_20260707-132641/report.md` | P1/P2/P3 全量网格 | | DSpark spec-tokens 对比 | `scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh` | `experiments/legacy_bench_results/dspark_st_comparison_20260707-150649/comparison_report.md` | `--spec-tokens 3` vs `5` | -| SGLang EAGLE vs DSpark | `scripts/start_sglang_dsv4_8card.sh` + 对比解析脚本 | `experiments/legacy_bench_results/eagle_grid/dspark_vs_eagle_report.md` | EAGLE 投机解码对比 | -| SGLang vs vLLM 后端对比（0707） | `scripts/benchmark_dsv4_backend_comparison.sh` | `experiments/legacy_bench_results/dsv4_backend_comparison_20260707/README.md` | 遗留 raw outputs 清单 | -| SGLang 8-card systematic | `scripts/run_sglang_benchmark.sh` | `experiments/legacy_bench_results/sglang_8card_systematic_20260704_120819/report.md` | 早期 SGLang 系统扫描 | -| SGLang 8-card max throughput | `scripts/run_sglang_max_throughput.sh` | `experiments/legacy_bench_results/sglang_8card_max_throughput_20260705_030839/summary.json` | 最大吞吐扫描 | -| vLLM-dspark Qwen3 | `scripts/bench_vllm_dspark_qwen3.py` | `experiments/legacy_bench_results/vllm_dspark_qwen3_20260705_121256/summary.json` | Qwen3 模型测速 | -| DSV4 后端对比（0705） | `scripts/bench_dsv4_comparison.py` | `experiments/legacy_bench_results/dsv4_comparison_20260705_152221/summary.json` | 早期 DSpark 配置对比 | ## 快速复现 @@ -83,15 +78,10 @@ bash experiments/dsv4_h200_vllm_dspark_vs_default/run_bench.sh bash experiments/dsv4_h200_max_context_length/run_bench.sh ``` -### DSpark grid（旧结构） +### DSpark grid / spec-tokens（旧结构） ```bash bash scripts/benchmark_dspark_0707/run_dspark_benchmark_grid.sh -``` - -### Spec-tokens 对比 - -```bash bash scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh ``` @@ -99,9 +89,6 @@ bash scripts/benchmark_dspark_0707/run_dspark_st_comparison.sh ```bash # 新结构 -python3 experiments/dsv4_p800_sglang/parse_results.py \ - experiments/dsv4_p800_sglang/results/ - python3 experiments/dsv4_h200_dspark/parse_results.py \ experiments/dsv4_h200_dspark/results/ diff --git a/docs/H200_QUICKSTART.md b/docs/H200_QUICKSTART.md index 664dee2..dc83a90 100644 --- a/docs/H200_QUICKSTART.md +++ b/docs/H200_QUICKSTART.md @@ -97,13 +97,7 @@ For a concrete example, see `experiments/dsv4_h200_dspark/`. ## 5. Cross-platform comparison -After running experiments on both P800 and H200, compare them with: - -```bash -python3 scripts/analysis/compare_experiments.py \ - experiments/dsv4_p800_sglang/results/ \ - experiments/dsv4_h200_dspark/results/ -``` +跨平台对比脚本目前未统一提供。可分别读取各实验 `results//results.json` 中的结构化数据，按 scenario 聚合后生成对比表。 ## Notes diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/sglang/results.json b/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/sglang/results.json new file mode 100644 index 0000000..926f365 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/sglang/results.json @@ -0,0 +1,42 @@ +{ + "metadata": { + "experiment": "dsv4_h200_long_context_matrix_sglang", + "run_id": "20260708-100016", + "timestamp": "2026-07-08T10:00:20+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "7e35945", + "git_dirty": "dirty", + "description": "H200 long-context matrix sglang TP=8 for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "sglang", + "server_start_script": "experiments/dsv4_h200_long_context_matrix/start_sglang.sh", + "groups": [ + { + "label": "64k", + "max_model_len": 80000, + "max_slots": 25 + }, + { + "label": "128k", + "max_model_len": 140000, + "max_slots": 10 + }, + { + "label": "256k", + "max_model_len": 270000, + "max_slots": 10 + } + ] + }, + "scenarios": [] +} diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/vllm/results.json b/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/vllm/results.json new file mode 100644 index 0000000..7a2af01 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260708-100016/vllm/results.json @@ -0,0 +1,42 @@ +{ + "metadata": { + "experiment": "dsv4_h200_long_context_matrix_vllm", + "run_id": "20260708-100016", + "timestamp": "2026-07-08T10:00:20+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "7e35945", + "git_dirty": "dirty", + "description": "H200 long-context matrix vllm TP=8 for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/dsv4_h200_long_context_matrix/start_vllm.sh", + "groups": [ + { + "label": "64k", + "max_model_len": 80000, + "max_slots": 25 + }, + { + "label": "128k", + "max_model_len": 140000, + "max_slots": 10 + }, + { + "label": "256k", + "max_model_len": 270000, + "max_slots": 10 + } + ] + }, + "scenarios": [] +} diff --git a/scripts/analysis/compare_experiments.py b/scripts/analysis/compare_experiments.py deleted file mode 100644 index 0c4d4ee..0000000 --- a/scripts/analysis/compare_experiments.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -"""Load results.json from multiple experiments and produce a comparison report. - -Usage: - python3 scripts/analysis/compare_experiments.py - python3 scripts/analysis/compare_experiments.py --output comparison.md -""" - -import argparse -import json -from collections import defaultdict -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -def load_results(): - """Glob experiments/*/results/*/results.json and load them.""" - results = [] - for path in sorted(REPO_ROOT.glob("experiments/*/results/*/results.json")): - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - data["_source"] = str(path.relative_to(REPO_ROOT)) - results.append(data) - except (json.JSONDecodeError, OSError) as e: - print(f"WARN: failed to load {path}: {e}") - return results - - -def main(): - parser = argparse.ArgumentParser(description="Compare benchmark experiments.") - parser.add_argument("--output", "-o", default="comparison_report.md", help="Output markdown file") - args = parser.parse_args() - - results = load_results() - if not results: - print("No experiments found under experiments/*/results/*/results.json") - return - - # Group by scenario name. - by_scenario = defaultdict(list) - for data in results: - meta = data.get("metadata", {}) - for scenario in data.get("scenarios", []): - key = scenario.get("name", "unknown") - by_scenario[key].append((meta, scenario)) - - output_path = Path(args.output) - with open(output_path, "w", encoding="utf-8") as f: - f.write("# Cross-Experiment Comparison\n\n") - f.write("| Experiment | Chip | Engine | Scenario | Conc | In/Out | Req/s | OutTok/s | TTFT p99 | TPOT p99 | E2E p99 |\n") - f.write("|---|---|---|---|---:|---:|---:|---:|---:|---:|---:|\n") - - for scenario_name in sorted(by_scenario.keys()): - for meta, scenario in by_scenario[scenario_name]: - lat = scenario.get("latencies", {}) - f.write( - f"| {meta.get('experiment', '')} " - f"| {meta.get('chip', '')} " - f"| {meta.get('engine', '')} " - f"| {scenario_name} " - f"| {scenario.get('concurrency', '')} " - f"| {scenario.get('input_len', '')}/{scenario.get('output_len', '')} " - f"| {scenario.get('request_throughput') or ''} " - f"| {scenario.get('output_token_throughput') or ''} " - f"| {lat.get('ttft_ms', {}).get('p99') or ''} " - f"| {lat.get('tpot_ms', {}).get('p99') or ''} " - f"| {lat.get('e2e_ms', {}).get('p99') or ''} |\n" - ) - - print(f"Loaded {len(results)} experiment runs.") - print(f"Wrote {output_path}") - - -if __name__ == "__main__": - main() diff --git a/scripts/analyze_dspark_nsys.py b/scripts/analyze_dspark_nsys.py deleted file mode 100644 index bd8ffb9..0000000 --- a/scripts/analyze_dspark_nsys.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -"""Analyze nsys traces for DSpark profiling. - -Usage: - python scripts/analyze_dspark_nsys.py -""" - -import json -import subprocess -import sys -from pathlib import Path - - -def run_nsys_report(rep_file: Path, report_name: str) -> str: - cmd = [ - "nsys", "stats", - "--report", report_name, - "--format", "json", - str(rep_file), - ] - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"nsys {report_name} failed: {result.stderr}") - return "" - return result.stdout - - -def summarize_cuda_sum(rep_file: Path) -> list[dict]: - """Return top CUDA kernels by total time.""" - text = run_nsys_report(rep_file, "cuda_kernel_sum") - if not text: - return [] - # nsys stats --format json outputs JSON after some header lines. - # Find the first '[' character. - start = text.find("[") - if start < 0: - return [] - data = json.loads(text[start:]) - # Sort by total time descending. - rows = data[1:] if len(data) > 1 else data - rows_sorted = sorted(rows, key=lambda x: x.get("Total Time (ns)", 0), reverse=True) - return rows_sorted[:50] - - -def summarize_nvtx_sum(rep_file: Path) -> list[dict]: - text = run_nsys_report(rep_file, "nvtx_sum") - start = text.find("[") - if start < 0: - return [] - data = json.loads(text[start:]) - rows = data[1:] if len(data) > 1 else data - rows_sorted = sorted(rows, key=lambda x: x.get("Total Time (ns)", 0), reverse=True) - return rows_sorted[:30] - - -def main(): - if len(sys.argv) < 2: - print("Usage: analyze_dspark_nsys.py ") - sys.exit(1) - rep_file = Path(sys.argv[1]) - if not rep_file.exists(): - print(f"File not found: {rep_file}") - sys.exit(1) - - print(f"Analyzing {rep_file}...\n") - - cuda_kernels = summarize_cuda_sum(rep_file) - print("=== Top CUDA Kernels by Total Time ===") - for row in cuda_kernels[:20]: - name = row.get("Name", "N/A") - total_ns = row.get("Total Time (ns)", 0) - count = row.get("Instances", 0) - avg_ns = row.get("Avg (ns)", 0) - print(f" {total_ns/1e6:8.2f} ms {count:6d} calls avg={avg_ns/1e6:6.3f} ms {name[:80]}") - - nvtx = summarize_nvtx_sum(rep_file) - if nvtx: - print("\n=== Top NVTX Ranges ===") - for row in nvtx[:20]: - name = row.get("Range", "N/A") - total_ns = row.get("Total Time (ns)", 0) - count = row.get("Instances", 0) - print(f" {total_ns/1e6:8.2f} ms {count:6d} calls {name[:80]}") - - -if __name__ == "__main__": - main() diff --git a/scripts/bench_dsv4_comparison.py b/scripts/bench_dsv4_comparison.py deleted file mode 100644 index e9acf5d..0000000 --- a/scripts/bench_dsv4_comparison.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -"""Benchmark orchestration for DeepSeek-V4 inference comparison. - -Compares: -- vllm-dspark + DeepSeek-V4-Flash-DSpark + DSpark (various spec-tokens) -- vllm-dspark + DeepSeek-V4-Flash-DSpark without spec decode -- vllm (0.24.0) + DeepSeek-V4-Flash without spec decode -- sglang + DeepSeek-V4-Flash + EAGLE (reuse existing results) -""" - -import json -import os -import subprocess -import sys -import time -import urllib.request -from datetime import datetime -from pathlib import Path - -ROOT = Path("/data/user1/yy") -RESULT_DIR = ROOT / "bench_results" / f"dsv4_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}" -RESULT_DIR.mkdir(parents=True, exist_ok=True) - -DATASET = "/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json" -NUM_PROMPTS = 200 -SEED = 42 -OUTPUT_LEN = 256 -HOST = "127.0.0.1" - -# Service configs to benchmark -SERVICES = [ - { - "name": "vllm-dspark-dspark-st5", - "engine": "vllm-dspark", - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_method": "dspark", - "spec_tokens": 5, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--spec-method", "dspark", - "--spec-model", "/data/models/DeepSeek-V4-Flash-DSpark", - "--spec-tokens", "5", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", "TMPDIR": str(ROOT / "tmp")}, - "bench_backend": "openai", - }, - { - "name": "vllm-dspark-nospec", - "engine": "vllm-dspark", - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_method": None, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", "TMPDIR": str(ROOT / "tmp")}, - "bench_backend": "openai", - }, - { - "name": "vllm-dspark-dspark-st3", - "engine": "vllm-dspark", - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_method": "dspark", - "spec_tokens": 3, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--spec-method", "dspark", - "--spec-model", "/data/models/DeepSeek-V4-Flash-DSpark", - "--spec-tokens", "3", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", "TMPDIR": str(ROOT / "tmp")}, - "bench_backend": "openai", - }, - { - "name": "vllm-dspark-dspark-st7", - "engine": "vllm-dspark", - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_method": "dspark", - "spec_tokens": 7, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--spec-method", "dspark", - "--spec-model", "/data/models/DeepSeek-V4-Flash-DSpark", - "--spec-tokens", "7", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", "TMPDIR": str(ROOT / "tmp")}, - "bench_backend": "openai", - }, - { - "name": "vllm-main-nospec", - "engine": "vllm", - "model": "/data/models/DeepSeek-V4-Flash", - "port": 30005, - "spec_method": None, - "cmd": [ - "/data/user1/yy/envs/vllm/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--disable-uvicorn-access-log", - "--port", "30005", - ], - "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", "TMPDIR": str(ROOT / "tmp")}, - "bench_backend": "openai", - }, -] - -CONCURRENCIES = [1, 16, 64] - - -def log(msg): - print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}", flush=True) - - -def wait_for_health(port, timeout=300): - url = f"http://{HOST}:{port}/health" - start = time.time() - while time.time() - start < timeout: - try: - with urllib.request.urlopen(url, timeout=5) as resp: - if resp.status == 200: - return True - except Exception: - pass - time.sleep(2) - return False - - -def start_service(service): - log(f"Starting {service['name']} on port {service['port']}...") - env = os.environ.copy() - env.update(service["env"]) - log_file = RESULT_DIR / f"{service['name']}_service.log" - proc = subprocess.Popen( - service["cmd"], - stdout=open(log_file, "w"), - stderr=subprocess.STDOUT, - env=env, - ) - if not wait_for_health(service["port"]): - log(f"ERROR: {service['name']} failed to start") - proc.terminate() - return None - log(f"{service['name']} is ready") - return proc - - -def stop_service(proc, name): - if proc is None: - return - log(f"Stopping {name} (pid {proc.pid})...") - proc.terminate() - try: - proc.wait(timeout=60) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - log(f"{name} stopped") - - -def run_benchmark(service, concurrency): - name = service["name"] - port = service["port"] - backend = service["bench_backend"] - result_file = RESULT_DIR / f"{name}_c{concurrency}.json" - log_file = RESULT_DIR / f"{name}_c{concurrency}.log" - - if service["engine"] == "vllm-dspark": - bench_cmd = [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "bench", "serve", - "--host", HOST, - "--port", str(port), - "--backend", backend, - "--dataset-name", "sharegpt", - "--dataset-path", DATASET, - "--sharegpt-output-len", str(OUTPUT_LEN), - "--num-prompts", str(NUM_PROMPTS), - "--max-concurrency", str(concurrency), - "--endpoint", "/v1/completions", - "--model", service["model"], - "--seed", str(SEED), - "--save-result", - "--result-dir", str(RESULT_DIR), - "--result-filename", result_file.name, - ] - else: - bench_cmd = [ - "/data/user1/yy/envs/vllm/bin/vllm", "bench", "serve", - "--host", HOST, - "--port", str(port), - "--backend", backend, - "--dataset-name", "sharegpt", - "--dataset-path", DATASET, - "--sharegpt-output-len", str(OUTPUT_LEN), - "--num-prompts", str(NUM_PROMPTS), - "--max-concurrency", str(concurrency), - "--endpoint", "/v1/completions", - "--model", service["model"], - "--seed", str(SEED), - "--save-result", - "--result-dir", str(RESULT_DIR), - "--result-filename", result_file.name, - ] - - log(f"Running benchmark {name} concurrency={concurrency}...") - start = time.time() - with open(log_file, "w") as f: - proc = subprocess.Popen(bench_cmd, stdout=f, stderr=subprocess.STDOUT) - proc.wait() - duration = time.time() - start - log(f"Benchmark {name} c={concurrency} finished in {duration:.1f}s, exit={proc.returncode}") - - if result_file.exists(): - with open(result_file) as f: - data = json.load(f) - return { - "service": name, - "engine": service["engine"], - "spec_method": service.get("spec_method"), - "spec_tokens": service.get("spec_tokens"), - "concurrency": concurrency, - "request_throughput": data.get("request_throughput"), - "output_throughput": data.get("output_throughput"), - "total_input_tokens": data.get("total_input_tokens"), - "total_output_tokens": data.get("total_output_tokens"), - "duration_s": data.get("duration_s"), - "result_file": str(result_file), - } - else: - log(f"WARNING: result file {result_file} not found") - return { - "service": name, - "engine": service["engine"], - "spec_method": service.get("spec_method"), - "spec_tokens": service.get("spec_tokens"), - "concurrency": concurrency, - "error": "result file missing", - "log_file": str(log_file), - } - - -def main(): - summary = [] - for service in SERVICES: - proc = start_service(service) - if proc is None: - continue - try: - for concurrency in CONCURRENCIES: - result = run_benchmark(service, concurrency) - summary.append(result) - # Save incremental summary - with open(RESULT_DIR / "summary.json", "w") as f: - json.dump(summary, f, indent=2) - finally: - stop_service(proc, service["name"]) - # Small gap between services - time.sleep(10) - - log(f"All benchmarks complete. Results in {RESULT_DIR}") - with open(RESULT_DIR / "summary.json", "w") as f: - json.dump(summary, f, indent=2) - - -if __name__ == "__main__": - main() diff --git a/scripts/bench_vllm_dspark_qwen3.py b/scripts/bench_vllm_dspark_qwen3.py deleted file mode 100644 index ff427d9..0000000 --- a/scripts/bench_vllm_dspark_qwen3.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 -"""Throughput benchmark for vLLM DSpark service (Qwen3-4B + dspark_qwen3_4b_block7).""" - -import argparse -import asyncio -import json -import random -import time -from dataclasses import dataclass, field -from datetime import datetime -from typing import List - -import aiohttp -import numpy as np -from transformers import AutoTokenizer - - -@dataclass -class RequestResult: - prompt_len: int = 0 - output_len: int = 0 - ttft_ms: float = 0.0 - tpot_ms: float = 0.0 - e2e_ms: float = 0.0 - success: bool = False - error: str = "" - - -async def async_request( - session: aiohttp.ClientSession, - url: str, - model_name: str, - prompt: str, - max_tokens: int, - prompt_len: int, - result: RequestResult, -) -> None: - payload = { - "model": model_name, - "prompt": prompt, - "max_tokens": max_tokens, - "temperature": 0.0, - "stream": True, - "stream_options": {"include_usage": True}, - } - result.prompt_len = prompt_len - start = time.perf_counter() - first_token_time = None - token_times: List[float] = [] - output_text = "" - try: - async with session.post(url, json=payload) as resp: - resp.raise_for_status() - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or line.startswith(":"): - continue - if line.startswith("data: "): - data = line[len("data: "):] - if data == "[DONE]": - break - chunk = json.loads(data) - choices = chunk.get("choices", []) - if choices: - delta = choices[0].get("text", "") - if delta: - now = time.perf_counter() - token_times.append(now) - output_text += delta - if first_token_time is None: - first_token_time = now - end = time.perf_counter() - result.e2e_ms = (end - start) * 1000.0 - if first_token_time is not None: - result.ttft_ms = (first_token_time - start) * 1000.0 - if len(token_times) >= 2: - # TPOT: average time between consecutive tokens - intervals = [ - token_times[i] - token_times[i - 1] - for i in range(1, len(token_times)) - ] - result.tpot_ms = sum(intervals) / len(intervals) * 1000.0 - result.output_len = len(token_times) - result.success = True - except Exception as e: - result.e2e_ms = (time.perf_counter() - start) * 1000.0 - result.error = str(e) - result.success = False - - -async def run_concurrency_benchmark( - url: str, - model_name: str, - prompts: List[str], - prompt_lens: List[int], - max_tokens: int, - concurrency: int, - num_prompts: int, - seed: int, -) -> dict: - rng = random.Random(seed) - indices = [rng.randrange(len(prompts)) for _ in range(num_prompts)] - - semaphore = asyncio.Semaphore(concurrency) - results: List[RequestResult] = [RequestResult() for _ in range(num_prompts)] - - async def bounded_request(idx: int, i: int): - async with semaphore: - async with aiohttp.ClientSession() as session: - await async_request( - session, - url, - model_name, - prompts[idx], - max_tokens, - prompt_lens[idx], - results[i], - ) - - start = time.perf_counter() - await asyncio.gather(*[bounded_request(idx, i) for i, idx in enumerate(indices)]) - duration = time.perf_counter() - start - - successes = [r for r in results if r.success] - failures = [r for r in results if not r.success] - if not successes: - return { - "concurrency": concurrency, - "num_prompts": num_prompts, - "duration_s": duration, - "success_count": 0, - "error": "all requests failed", - "errors": [r.error for r in failures[:5]], - } - - total_in_tokens = sum(r.prompt_len for r in successes) - total_out_tokens = sum(r.output_len for r in successes) - total_tokens = total_in_tokens + total_out_tokens - - ttfts = [r.ttft_ms for r in successes] - tpots = [r.tpot_ms for r in successes] - e2es = [r.e2e_ms for r in successes] - - return { - "concurrency": concurrency, - "num_prompts": num_prompts, - "duration_s": duration, - "success_count": len(successes), - "fail_count": len(failures), - "input_tokens": total_in_tokens, - "output_tokens": total_out_tokens, - "total_tokens": total_tokens, - "request_throughput": len(successes) / duration, - "input_throughput": total_in_tokens / duration, - "output_throughput": total_out_tokens / duration, - "total_throughput": total_tokens / duration, - "mean_ttft_ms": float(np.mean(ttfts)), - "p50_ttft_ms": float(np.percentile(ttfts, 50)), - "p99_ttft_ms": float(np.percentile(ttfts, 99)), - "mean_tpot_ms": float(np.mean(tpots)), - "p50_tpot_ms": float(np.percentile(tpots, 50)), - "p99_tpot_ms": float(np.percentile(tpots, 99)), - "mean_e2e_ms": float(np.mean(e2es)), - "p50_e2e_ms": float(np.percentile(e2es, 50)), - "p99_e2e_ms": float(np.percentile(e2es, 99)), - "errors": [r.error for r in failures[:5]], - } - - -def load_sharegpt_prompts(path: str, tokenizer, max_samples: int = 2000): - with open(path, "r") as f: - data = json.load(f) - prompts = [] - lens = [] - for item in data: - conv = item.get("conversations", []) - for turn in conv: - if turn.get("from") == "human": - prompt = turn.get("value", "") - if prompt: - prompts.append(prompt) - lens.append(len(tokenizer.encode(prompt))) - break - if len(prompts) >= max_samples: - break - return prompts, lens - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30003) - parser.add_argument("--model", default="/data/models/Qwen3-4B") - parser.add_argument("--dataset", default="/data/user1/yy/datasets/ShareGPT_filtered_chat.json") - parser.add_argument("--tokenizer", default="/data/models/Qwen3-4B") - parser.add_argument("--max-tokens", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=500) - parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4, 16, 64, 128]) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output-dir", default="/data/user1/yy/bench_results") - args = parser.parse_args() - - url = f"http://{args.host}:{args.port}/v1/completions" - print(f"Loading tokenizer from {args.tokenizer} ...") - tokenizer = AutoTokenizer.from_pretrained(args.tokenizer) - print(f"Loading dataset from {args.dataset} ...") - prompts, prompt_lens = load_sharegpt_prompts(args.dataset, tokenizer) - print(f"Loaded {len(prompts)} prompts, prompt lens: min={min(prompt_lens)}, max={max(prompt_lens)}, mean={sum(prompt_lens)/len(prompt_lens):.1f}") - - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - result_dir = f"{args.output_dir}/vllm_dspark_qwen3_{timestamp}" - import os - os.makedirs(result_dir, exist_ok=True) - summary_file = f"{result_dir}/summary.json" - - summary = { - "model": args.model, - "draft_model": "deepseek-ai/dspark_qwen3_4b_block7", - "url": url, - "max_tokens": args.max_tokens, - "num_prompts": args.num_prompts, - "dataset": args.dataset, - "timestamp": timestamp, - "results": [], - } - - print("\nStarting benchmark sweeps...") - print(f"{'Concurrency':>12} {'Req/s':>10} {'In tok/s':>12} {'Out tok/s':>12} {'Total tok/s':>13} {'TTFT(ms)':>10} {'TPOT(ms)':>10} {'E2E(ms)':>10}") - print("-" * 100) - - for c in args.concurrency: - print(f"Running concurrency={c} ...", flush=True) - result = asyncio.run( - run_concurrency_benchmark( - url, - args.model, - prompts, - prompt_lens, - args.max_tokens, - c, - args.num_prompts, - args.seed, - ) - ) - summary["results"].append(result) - with open(summary_file, "w") as f: - json.dump(summary, f, indent=2) - if result.get("success_count", 0) == 0: - print(f"{c:>12} FAILED: {result.get('error', 'unknown')}") - continue - print( - f"{c:>12} " - f"{result['request_throughput']:>10.2f} " - f"{result['input_throughput']:>12.2f} " - f"{result['output_throughput']:>12.2f} " - f"{result['total_throughput']:>13.2f} " - f"{result['mean_ttft_ms']:>10.1f} " - f"{result['mean_tpot_ms']:>10.1f} " - f"{result['mean_e2e_ms']:>10.1f}" - ) - - print(f"\nSummary saved to {summary_file}") - - -if __name__ == "__main__": - main() diff --git a/scripts/benchmark_dsv4_backend_comparison.sh b/scripts/benchmark_dsv4_backend_comparison.sh deleted file mode 100644 index 5e7561e..0000000 --- a/scripts/benchmark_dsv4_backend_comparison.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# Reproducible SGLang vs vLLM backend comparison for DeepSeek-V4-Flash. -# Produces JSONL raw outputs compatible with the legacy 2026-07-07 sweep. -# -# Usage: -# bash scripts/benchmark_dsv4_backend_comparison.sh [vllm|sglang|all] -# -# Output: -# bench_results/dsv4_backend_comparison_${RUN_ID}/raw_outputs/ - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" - -BACKEND_FILTER="${1:-all}" -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${ROOT_DIR}/bench_results/dsv4_backend_comparison_${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -mkdir -p "${RAW_DIR}" "${LOG_DIR}" - -BENCH_PY="${ROOT_DIR}/envs/sglang/bin/python" -DATASET="${ROOT_DIR}/datasets/ShareGPT_filtered_chat.json" -MODEL="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" -NUM_PROMPTS=512 -WARMUP=100 - -log() { - echo "[$(date --iso-8601=seconds)] $*" | tee -a "${LOG_DIR}/orchestrator.log" -} - -run_case() { - local backend="$1" - local port="$2" - local concurrency="$3" - local input_len="$4" - local output_len="$5" - local output_file="${RAW_DIR}/${backend}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - local detail_log="${LOG_DIR}/${backend}_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running ${backend}: concurrency=${concurrency} input=${input_len} output=${output_len} -> ${output_file}" - - "${BENCH_PY}" -m sglang.bench_serving \ - --backend "${backend}" \ - --host 127.0.0.1 \ - --port "${port}" \ - --model "${MODEL}" \ - --served-model-name "${SERVED_MODEL_NAME}" \ - --dataset-name random \ - --dataset-path "${DATASET}" \ - --random-input-len "${input_len}" \ - --random-output-len "${output_len}" \ - --num-prompts "${NUM_PROMPTS}" \ - --max-concurrency "${concurrency}" \ - --warmup-requests "${WARMUP}" \ - --output-file "${output_file}" \ - --output-details \ - > "${detail_log}" 2>&1 - - log "finished ${backend} c=${concurrency} i=${input_len} o=${output_len}" -} - -main() { - log "run_id=${RUN_ID}" - log "result_root=${RESULT_ROOT}" - log "backend_filter=${BACKEND_FILTER}" - - # This script assumes a healthy server is already running on the target port. - if [[ "${BACKEND_FILTER}" == "all" || "${BACKEND_FILTER}" == "sglang" ]]; then - log "===== SGLang sweep (port 30000) =====" - run_case sglang 30000 32 512 256 - run_case sglang 30000 128 512 256 - run_case sglang 30000 256 512 256 - run_case sglang 30000 512 512 256 - run_case sglang 30000 512 512 2000 - run_case sglang 30000 32 4000 512 - run_case sglang 30000 128 4000 512 - run_case sglang 30000 512 4000 512 - run_case sglang 30000 32 16000 512 - run_case sglang 30000 128 16000 512 - run_case sglang 30000 512 1000 256 - run_case sglang 30000 768 1000 256 - run_case sglang 30000 1024 1000 256 - run_case sglang 30000 512 1000 1000 - fi - - if [[ "${BACKEND_FILTER}" == "all" || "${BACKEND_FILTER}" == "vllm" ]]; then - log "===== vLLM sweep (port 8000) =====" - run_case vllm 8000 32 512 256 - run_case vllm 8000 128 512 256 - run_case vllm 8000 256 512 256 - run_case vllm 8000 128 512 2000 - run_case vllm 8000 256 512 2000 - run_case vllm 8000 512 512 2000 - run_case vllm 8000 32 1000 256 - run_case vllm 8000 128 1000 256 - run_case vllm 8000 256 1000 256 - run_case vllm 8000 512 1000 256 - run_case vllm 8000 768 1000 256 - run_case vllm 8000 1024 1000 256 - run_case vllm 8000 32 1000 1000 - run_case vllm 8000 128 1000 1000 - run_case vllm 8000 256 1000 1000 - run_case vllm 8000 512 1000 1000 - run_case vllm 8000 1024 1000 1000 - run_case vllm 8000 32 4000 512 - run_case vllm 8000 128 4000 512 - run_case vllm 8000 256 4000 512 - run_case vllm 8000 512 4000 512 - run_case vllm 8000 32 8000 1000 - run_case vllm 8000 128 8000 1000 - run_case vllm 8000 256 8000 1000 - run_case vllm 8000 512 8000 1000 - run_case vllm 8000 32 16000 512 - run_case vllm 8000 128 16000 512 - run_case vllm 8000 256 16000 512 - run_case vllm 8000 32 32000 512 - run_case vllm 8000 128 32000 512 - fi - - log "===== Done =====" - log "all outputs saved to ${RAW_DIR}" -} - -main "$@" diff --git a/scripts/install_vllm_dspark.sh b/scripts/install_vllm_dspark.sh deleted file mode 100755 index d1595f4..0000000 --- a/scripts/install_vllm_dspark.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -e - -export TMPDIR=/data/tmp -export TEMP=/data/tmp -export TMP=/data/tmp -export PIP_CACHE_DIR=/data/tmp/pip-cache -export SETUPTOOLS_SCM_PRETEND_VERSION=0.23.1.dev788 -mkdir -p $PIP_CACHE_DIR - -cd /data/user1/yy/vllm-main -PYTHON=/data/user1/yy/envs/vllm-dspark/bin/python -PIP=/data/user1/yy/envs/vllm-dspark/bin/pip - -echo "Installing torch 2.11.0 with CUDA 12.9..." -$PIP install torch==2.11.0 --index-url https://download.pytorch.org/whl/cu129 --cache-dir $PIP_CACHE_DIR - -echo "Installing numpy and build dependencies..." -$PIP install numpy cmake ninja packaging "setuptools>=77.0.3,<81.0.0" setuptools-scm setuptools-rust wheel jinja2 --cache-dir $PIP_CACHE_DIR - -echo "Installing vllm from source (this will compile CUDA kernels, may take 30-60 min)..." -$PIP install -e . --no-build-isolation --cache-dir $PIP_CACHE_DIR - -echo "Verifying installation..." -$PYTHON -c "import vllm; print('vllm version:', vllm.__version__)" - -echo "Installation complete." diff --git a/scripts/profile_dspark.py b/scripts/profile_dspark.py deleted file mode 100755 index ce40591..0000000 --- a/scripts/profile_dspark.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -"""Profile DSpark inference with Nsight Systems and/or PyTorch profiler. - -Usage: - python scripts/profile_dspark.py --config dspark-st3 --concurrency 64 --tool nsys - python scripts/profile_dspark.py --config nospec --concurrency 64 --tool nsys -""" - -import argparse -import json -import os -import subprocess -import sys -import time -import urllib.request -from datetime import datetime -from pathlib import Path - -ROOT = Path("/data/user1/yy") -RESULT_DIR = ROOT / "bench_results" / f"dspark_profile_{datetime.now().strftime('%Y%m%d_%H%M%S')}" -RESULT_DIR.mkdir(parents=True, exist_ok=True) - -DATASET = "/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json" -NUM_PROMPTS = 200 -SEED = 42 -OUTPUT_LEN = 256 -HOST = "127.0.0.1" - -CONFIGS = { - "dspark-st3": { - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_tokens": 3, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--gpu-memory-utilization", "0.75", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--spec-method", "dspark", - "--spec-model", "/data/models/DeepSeek-V4-Flash-DSpark", - "--spec-tokens", "3", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - }, - "dspark-st5": { - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "spec_tokens": 5, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--gpu-memory-utilization", "0.75", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--spec-method", "dspark", - "--spec-model", "/data/models/DeepSeek-V4-Flash-DSpark", - "--spec-tokens", "5", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - }, - "nospec": { - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "port": 30004, - "cmd": [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "serve", - "/data/models/DeepSeek-V4-Flash-DSpark", - "--trust-remote-code", - "--tensor-parallel-size", "8", - "--kv-cache-dtype", "fp8", - "--block-size", "256", - "--max-model-len", "auto", - "--max-num-seqs", "256", - "--gpu-memory-utilization", "0.75", - "--tokenizer-mode", "deepseek_v4", - "--reasoning-parser", "deepseek_v4", - "--no-disable-hybrid-kv-cache-manager", - "--disable-uvicorn-access-log", - "--port", "30004", - ], - }, -} - - -def log(msg): - print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}", flush=True) - - -def wait_for_health(port, timeout=300): - url = f"http://{HOST}:{port}/health" - start = time.time() - while time.time() - start < timeout: - try: - with urllib.request.urlopen(url, timeout=5) as resp: - if resp.status == 200: - return True - except Exception: - pass - time.sleep(2) - return False - - -def run_warmup(port, model, n=20): - """Run a few warmup requests to stabilize CUDA graphs.""" - log(f"Running {n} warmup requests...") - for i in range(n): - try: - req = urllib.request.Request( - f"http://{HOST}:{port}/v1/completions", - data=json.dumps({ - "model": model, - "prompt": "Hello, how are you?", - "max_tokens": 32, - "temperature": 0, - "seed": SEED, - }).encode(), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=60) as resp: - resp.read() - except Exception as e: - log(f"Warmup request {i} failed: {e}") - - -def run_benchmark(port, model, concurrency, result_dir): - result_file = result_dir / f"profile_c{concurrency}.json" - log_file = result_dir / f"profile_c{concurrency}.log" - bench_cmd = [ - "/data/user1/yy/envs/vllm-dspark/bin/vllm", "bench", "serve", - "--host", HOST, - "--port", str(port), - "--backend", "openai", - "--dataset-name", "sharegpt", - "--dataset-path", DATASET, - "--sharegpt-output-len", str(OUTPUT_LEN), - "--num-prompts", str(NUM_PROMPTS), - "--max-concurrency", str(concurrency), - "--endpoint", "/v1/completions", - "--model", model, - "--seed", str(SEED), - "--save-result", - "--result-dir", str(result_dir), - "--result-filename", result_file.name, - ] - log(f"Running benchmark c={concurrency}...") - start = time.time() - with open(log_file, "w") as f: - proc = subprocess.Popen(bench_cmd, stdout=f, stderr=subprocess.STDOUT) - proc.wait() - duration = time.time() - start - log(f"Benchmark c={concurrency} finished in {duration:.1f}s, exit={proc.returncode}") - return result_file - - -def start_service(config, tool, result_dir): - name = config - cfg = CONFIGS[config] - port = cfg["port"] - base_cmd = cfg["cmd"] - - if tool == "nsys": - nsys_output = result_dir / f"{name}_nsys" - cmd = [ - "nsys", "profile", - "--sample=none", - "--cpuctxsw=none", - "--trace=cuda,nvtx,osrt", - "--output", str(nsys_output), - "--force-overwrite", "true", - "--delay", "30", - "--duration", "15", - ] + base_cmd - else: - cmd = base_cmd - - log(f"Starting service for {name} (tool={tool}) on port {port}...") - log_file = result_dir / f"{name}_service.log" - env = os.environ.copy() - env["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7" - env["TMPDIR"] = str(ROOT / "tmp") - - proc = subprocess.Popen( - cmd, - stdout=open(log_file, "w"), - stderr=subprocess.STDOUT, - env=env, - ) - if not wait_for_health(port): - log(f"ERROR: {name} failed to start") - proc.terminate() - return None - log(f"{name} is ready") - return proc - - -def stop_service(proc, name): - if proc is None: - return - log(f"Stopping {name} (pid {proc.pid})...") - proc.terminate() - try: - proc.wait(timeout=60) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - log(f"{name} stopped") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--config", required=True, choices=list(CONFIGS.keys())) - parser.add_argument("--concurrency", type=int, default=64) - parser.add_argument("--tool", default="nsys", choices=["nsys"]) - parser.add_argument("--skip-bench", action="store_true", help="Only capture service startup/warmup, skip benchmark") - args = parser.parse_args() - - result_dir = RESULT_DIR / f"{args.config}_c{args.concurrency}_{args.tool}" - result_dir.mkdir(parents=True, exist_ok=True) - log(f"Results will be saved to {result_dir}") - - proc = start_service(args.config, args.tool, result_dir) - if proc is None: - sys.exit(1) - - try: - run_warmup(CONFIGS[args.config]["port"], CONFIGS[args.config]["model"], n=30) - if not args.skip_bench: - run_benchmark( - CONFIGS[args.config]["port"], - CONFIGS[args.config]["model"], - args.concurrency, - result_dir, - ) - else: - log("--skip-bench set; sleeping 60s to capture warmup/steady state...") - time.sleep(60) - finally: - stop_service(proc, args.config) - - log(f"All done. Results in {result_dir}") - - -if __name__ == "__main__": - main() diff --git a/scripts/run_sglang_benchmark.sh b/scripts/run_sglang_benchmark.sh deleted file mode 100755 index bb19967..0000000 --- a/scripts/run_sglang_benchmark.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Comprehensive benchmark for sglang 8-card DeepSeek-V4-Flash -set -e - -cd /data/user1/yy -mkdir -p bench_results logs - -PYTHON="/data/user1/yy/envs/sglang/bin/python" -BENCH="-m sglang.bench_serving" -HOST="127.0.0.1" -PORT="30000" -MODEL="/data/models/DeepSeek-V4-Flash" -DATASET="/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json" -NUM_PROMPTS=2000 -SEED=42 - -RESULT_DIR="bench_results/sglang_8card_systematic_$(date +%Y%m%d_%H%M%S)" -mkdir -p "${RESULT_DIR}" - -SUMMARY_FILE="${RESULT_DIR}/summary.json" -LOG_FILE="${RESULT_DIR}/benchmark.log" - -# Use a marker file for summary accumulation -SUMMARY_TMP="${RESULT_DIR}/.summary_tmp.json" -echo '[]' > "${SUMMARY_TMP}" - -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}" -} - -wait_for_server() { - log "Waiting for sglang server at ${HOST}:${PORT} to become ready..." - local retries=0 - local max_retries=120 - while ! curl -s "http://${HOST}:${PORT}/v1/models" > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ "${retries}" -ge "${max_retries}" ]; then - log "ERROR: Server not ready after ${max_retries} retries" - exit 1 - fi - sleep 2 - done - log "Server is ready." -} - -run_bench() { - local name="$1" - shift - local output_file="${RESULT_DIR}/${name}.json" - local detail_log="${RESULT_DIR}/${name}.log" - - log "---------------------------------------------------" - log "Running benchmark: ${name}" - log "Args: $*" - log "Output: ${output_file}" - - ${PYTHON} ${BENCH} \ - --backend sglang \ - --host "${HOST}" \ - --port "${PORT}" \ - --dataset-name sharegpt \ - --dataset-path "${DATASET}" \ - --num-prompts "${NUM_PROMPTS}" \ - --model "${MODEL}" \ - --output-file "${output_file}" \ - --output-details \ - --seed "${SEED}" \ - "$@" \ - > "${detail_log}" 2>&1 - - local status=$? - if [ ${status} -ne 0 ]; then - log "ERROR: Benchmark ${name} failed with exit code ${status}" - return ${status} - fi - - log "Benchmark ${name} completed successfully." - - # Extract key metrics from the log - local req_throughput=$(grep -oP 'Request throughput \(req/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local input_throughput=$(grep -oP 'Input token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local output_throughput=$(grep -oP 'Output token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local total_throughput=$(grep -oP 'Total token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_ttft=$(grep -oP 'Mean TTFT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_tpot=$(grep -oP 'Mean TPOT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_e2e=$(grep -oP 'Mean E2E Latency \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local duration=$(grep -oP 'Benchmark duration \(s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local accept_length=$(grep -oP 'Accept length:\s+\K[0-9.]+' "${detail_log}" || echo "null") - - # Append to summary tmp - ${PYTHON} - <8} {'In tok/s':>10} {'Out tok/s':>10} {'Total tok/s':>12} {'TTFT':>8} {'TPOT':>8} {'E2E':>10}") -print("-" * 110) -for entry in data: - print(f"{entry['name']:<30} {entry['request_throughput']:>8.2f} {entry['input_token_throughput']:>10.2f} {entry['output_token_throughput']:>10.2f} {entry['total_token_throughput']:>12.2f} {entry['mean_ttft_ms']:>8.1f} {entry['mean_tpot_ms']:>8.1f} {entry['mean_e2e_latency_ms']:>10.1f}") -PYEOF -} - -main "$@" diff --git a/scripts/run_sglang_max_throughput.sh b/scripts/run_sglang_max_throughput.sh deleted file mode 100755 index 7d1513b..0000000 --- a/scripts/run_sglang_max_throughput.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -# Max throughput benchmark for sglang 8-card DeepSeek-V4-Flash -set -e - -cd /data/user1/yy -mkdir -p bench_results logs - -PYTHON="/data/user1/yy/envs/sglang/bin/python" -BENCH="-m sglang.bench_serving" -HOST="127.0.0.1" -PORT="30000" -MODEL="/data/models/DeepSeek-V4-Flash" -DATASET="/data/user1/yy/datasets/ShareGPT_V4.3_unfiltered_cleaned_split.json" -NUM_PROMPTS=2000 -SEED=42 - -RESULT_DIR="bench_results/sglang_8card_max_throughput_$(date +%Y%m%d_%H%M%S)" -mkdir -p "${RESULT_DIR}" - -SUMMARY_FILE="${RESULT_DIR}/summary.json" -LOG_FILE="${RESULT_DIR}/benchmark.log" -SUMMARY_TMP="${RESULT_DIR}/.summary_tmp.json" -echo '[]' > "${SUMMARY_TMP}" - -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}" -} - -wait_for_server() { - log "Waiting for sglang server at ${HOST}:${PORT} to become ready..." - local retries=0 - local max_retries=120 - while ! curl -s "http://${HOST}:${PORT}/v1/models" > /dev/null 2>&1; do - retries=$((retries + 1)) - if [ "${retries}" -ge "${max_retries}" ]; then - log "ERROR: Server not ready after ${max_retries} retries" - exit 1 - fi - sleep 2 - done - log "Server is ready." -} - -run_bench() { - local name="$1" - shift - local output_file="${RESULT_DIR}/${name}.json" - local detail_log="${RESULT_DIR}/${name}.log" - - log "---------------------------------------------------" - log "Running benchmark: ${name}" - log "Args: $*" - log "Output: ${output_file}" - - ${PYTHON} ${BENCH} \ - --backend sglang \ - --host "${HOST}" \ - --port "${PORT}" \ - --dataset-name sharegpt \ - --dataset-path "${DATASET}" \ - --num-prompts "${NUM_PROMPTS}" \ - --model "${MODEL}" \ - --output-file "${output_file}" \ - --output-details \ - --seed "${SEED}" \ - "$@" \ - > "${detail_log}" 2>&1 - - local status=$? - if [ ${status} -ne 0 ]; then - log "ERROR: Benchmark ${name} failed with exit code ${status}" - return ${status} - fi - - log "Benchmark ${name} completed successfully." - - local req_throughput=$(grep -oP 'Request throughput \(req/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local input_throughput=$(grep -oP 'Input token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local output_throughput=$(grep -oP 'Output token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local total_throughput=$(grep -oP 'Total token throughput \(tok/s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_ttft=$(grep -oP 'Mean TTFT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_tpot=$(grep -oP 'Mean TPOT \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local mean_e2e=$(grep -oP 'Mean E2E Latency \(ms\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local duration=$(grep -oP 'Benchmark duration \(s\):\s+\K[0-9.]+' "${detail_log}" || echo "null") - local accept_length=$(grep -oP 'Accept length:\s+\K[0-9.]+' "${detail_log}" || echo "null") - - ${PYTHON} - <8} {'In tok/s':>10} {'Out tok/s':>10} {'Total tok/s':>12} {'TTFT':>8} {'TPOT':>8} {'E2E':>10}") -print("-" * 110) -for entry in data: - print(f"{entry['name']:<30} {entry['request_throughput']:>8.2f} {entry['input_token_throughput']:>10.2f} {entry['output_token_throughput']:>10.2f} {entry['total_token_throughput']:>12.2f} {entry['mean_ttft_ms']:>8.1f} {entry['mean_tpot_ms']:>8.1f} {entry['mean_e2e_latency_ms']:>10.1f}") -PYEOF -} - -main "$@" diff --git a/scripts/start_dsv4_dspark_8card_bf16kv.sh b/scripts/start_dsv4_dspark_8card_bf16kv.sh deleted file mode 100755 index 3833a53..0000000 --- a/scripts/start_dsv4_dspark_8card_bf16kv.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-dspark" -VLLM="$VENV/bin/vllm" - -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -PORT=30004 -TP=8 -LOG="/data/user1/yy/logs/dsv4_flash_dspark_tp${TP}_bf16kv_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash-DSpark (TP=$TP, kv=bf16) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" -nohup "$VLLM" serve "$MODEL" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --kv-cache-dtype bfloat16 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 256 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --spec-method dspark \ - --spec-model "$MODEL" \ - --spec-tokens 5 \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port "$PORT" \ - > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -100 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -100 "$LOG" -exit 1 diff --git a/scripts/start_dsv4_dspark_8card_flashinfer.sh b/scripts/start_dsv4_dspark_8card_flashinfer.sh deleted file mode 100755 index 458e309..0000000 --- a/scripts/start_dsv4_dspark_8card_flashinfer.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-dspark" -VLLM="$VENV/bin/vllm" -export PATH="$VENV/bin:$PATH" - -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -PORT=30004 -TP=8 -LOG="/data/user1/yy/logs/dsv4_flash_dspark_tp${TP}_flashinfer_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash-DSpark (TP=$TP, FlashInfer) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" -nohup "$VLLM" serve "$MODEL" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 256 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --attention-backend FLASHINFER_MLA_SPARSE_DSV4 \ - --spec-method dspark \ - --spec-model "$MODEL" \ - --spec-tokens 5 \ - --speculative-config '{"attention_backend": "FLASHINFER_MLA_SPARSE_DSV4"}' \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port "$PORT" \ - > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -100 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -100 "$LOG" -exit 1 diff --git a/scripts/start_dsv4_dspark_8card_param.sh b/scripts/start_dsv4_dspark_8card_param.sh deleted file mode 100755 index 6be3ac5..0000000 --- a/scripts/start_dsv4_dspark_8card_param.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-dspark" -VLLM="$VENV/bin/vllm" -export PATH="$VENV/bin:$PATH" - -MODEL="/data/models/DeepSeek-V4-Flash-DSpark" -PORT="${PORT:-30004}" -TP="${TP:-8}" -SPEC_TOKENS="${SPEC_TOKENS:-5}" -KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" -ATTENTION_BACKEND="${ATTENTION_BACKEND:-}" -SPECULATIVE_CONFIG="${SPECULATIVE_CONFIG:-}" - -LOG_TAG="tp${TP}_${KV_CACHE_DTYPE}_st${SPEC_TOKENS}" -if [[ -n "$ATTENTION_BACKEND" ]]; then - LOG_TAG="${LOG_TAG}_${ATTENTION_BACKEND}" -fi -LOG="/data/user1/yy/logs/dsv4_flash_dspark_${LOG_TAG}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/dsv4_flash_dspark.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash-DSpark (TP=$TP, spec-tokens=$SPEC_TOKENS, kv=$KV_CACHE_DTYPE) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" - -cmd=( - "$VLLM" serve "$MODEL" - --trust-remote-code - --tensor-parallel-size "$TP" - --kv-cache-dtype "$KV_CACHE_DTYPE" - --block-size 256 - --max-model-len auto - --max-num-seqs 256 - --tokenizer-mode deepseek_v4 - --reasoning-parser deepseek_v4 - --spec-method dspark - --spec-model "$MODEL" - --spec-tokens "$SPEC_TOKENS" - --no-disable-hybrid-kv-cache-manager - --disable-uvicorn-access-log - --port "$PORT" -) - -if [[ -n "$ATTENTION_BACKEND" ]]; then - cmd+=(--attention-backend "$ATTENTION_BACKEND") -fi - -if [[ -n "$SPECULATIVE_CONFIG" ]]; then - cmd+=(--speculative-config "$SPECULATIVE_CONFIG") -fi - -nohup "${cmd[@]}" > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -100 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -100 "$LOG" -exit 1 diff --git a/scripts/start_pd_single_node.sh b/scripts/start_pd_single_node.sh deleted file mode 100755 index 3f3a2fd..0000000 --- a/scripts/start_pd_single_node.sh +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -set -e - -# sglang PD 分离单节点启动脚本 -# 104 上前 4 张卡做 prefill，后 4 张卡做 decode -# 使用 NIXL/UCX 作为传输后端（Mooncake 无法安装） - -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -BOOTSTRAP_PORT=30010 -PREFILL_PORT=30001 -DECODE_PORT=30002 -LB_PORT=30000 -HOST="0.0.0.0" - -VENV="/data/user1/yy/envs/sglang" -PYTHON="$VENV/bin/python" -LOG_DIR="/data/user1/yy/pd_logs" -mkdir -p "$LOG_DIR" - -# NIXL/UCX 配置：单节点内优先用 CUDA IPC / shared memory，禁用 IB -export SGLANG_DISAGGREGATION_NIXL_BACKEND="UCX" -export UCX_TLS="self,sm,cuda_copy,cuda_ipc,tcp" -export UCX_NET_DEVICES="" - -echo "=== Starting sglang PD disaggregation (single node) ===" -echo "Model: $MODEL_PATH" -echo "Prefill GPUs: 0-3, port $PREFILL_PORT" -echo "Decode GPUs: 4-7, port $DECODE_PORT" -echo "Load balancer port: $LB_PORT" -echo "Logs: $LOG_DIR" -echo "" - -# Clean up old logs -rm -f "$LOG_DIR"/*.log - -# Common args for both prefill and decode -COMMON_ARGS=( - --trust-remote-code - --model-path "$MODEL_PATH" - --tp 4 - --moe-runner-backend marlin - --disable-cuda-graph - --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" - --disaggregation-transfer-backend nixl - --host "$HOST" -) - -# Start prefill server (GPUs 0-3) -echo "[1/3] Starting prefill server on GPUs 0-3..." -CUDA_VISIBLE_DEVICES=0,1,2,3 nohup "$PYTHON" -m sglang.launch_server \ - "${COMMON_ARGS[@]}" \ - --disaggregation-mode prefill \ - --port "$PREFILL_PORT" \ - > "$LOG_DIR/prefill.log" 2>&1 & -PREFILL_PID=$! -echo "Prefill PID: $PREFILL_PID" - -# Wait for prefill health -for i in {1..120}; do - if curl -s "http://127.0.0.1:$PREFILL_PORT/health" > /dev/null 2>&1; then - echo "Prefill server is ready" - break - fi - if ! kill -0 $PREFILL_PID 2>/dev/null; then - echo "ERROR: Prefill server exited early" - tail -50 "$LOG_DIR/prefill.log" - exit 1 - fi - echo "Waiting for prefill server... ($i/120)" - sleep 5 -done - -# Start decode server (GPUs 4-7) -echo "[2/3] Starting decode server on GPUs 4-7..." -CUDA_VISIBLE_DEVICES=0,1,2,3 nohup "$PYTHON" -m sglang.launch_server \ - "${COMMON_ARGS[@]}" \ - --disaggregation-mode decode \ - --base-gpu-id 4 \ - --port "$DECODE_PORT" \ - > "$LOG_DIR/decode.log" 2>&1 & -DECODE_PID=$! -echo "Decode PID: $DECODE_PID" - -# Wait for decode health -for i in {1..120}; do - if curl -s "http://127.0.0.1:$DECODE_PORT/health" > /dev/null 2>&1; then - echo "Decode server is ready" - break - fi - if ! kill -0 $DECODE_PID 2>/dev/null; then - echo "ERROR: Decode server exited early" - tail -50 "$LOG_DIR/decode.log" - exit 1 - fi - echo "Waiting for decode server... ($i/120)" - sleep 5 -done - -# Start PD load balancer -echo "[3/3] Starting PD load balancer on port $LB_PORT..." -nohup "$PYTHON" -m sglang_router.launch_router \ - --pd-disaggregation \ - --mini-lb \ - --prefill "http://127.0.0.1:$PREFILL_PORT" \ - --decode "http://127.0.0.1:$DECODE_PORT" \ - --host "$HOST" \ - --port "$LB_PORT" \ - > "$LOG_DIR/lb.log" 2>&1 & -LB_PID=$! -echo "LB PID: $LB_PID" - -# Wait for LB health -for i in {1..60}; do - if curl -s "http://127.0.0.1:$LB_PORT/health" > /dev/null 2>&1; then - echo "Load balancer is ready" - break - fi - if ! kill -0 $LB_PID 2>/dev/null; then - echo "ERROR: Load balancer exited early" - tail -50 "$LOG_DIR/lb.log" - exit 1 - fi - echo "Waiting for load balancer... ($i/60)" - sleep 2 -done - -echo "" -echo "=== PD disaggregation is ready ===" -echo "Load balancer URL: http://127.0.0.1:$LB_PORT" -echo "Prefill URL: http://127.0.0.1:$PREFILL_PORT" -echo "Decode URL: http://127.0.0.1:$DECODE_PORT" -echo "" -echo "Test command:" -echo " curl http://127.0.0.1:$LB_PORT/v1/completions \\" -echo " -H \"Content-Type: application/json\" \\" -echo " -d '{\"model\":\"DeepSeek-V4-Flash\",\"prompt\":\"Hello\",\"max_tokens\":32}'" -echo "" -echo "To stop: kill $PREFILL_PID $DECODE_PID $LB_PID" diff --git a/scripts/start_sglang_dsv4_8card.sh b/scripts/start_sglang_dsv4_8card.sh deleted file mode 100755 index 638ecbf..0000000 --- a/scripts/start_sglang_dsv4_8card.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Start sglang 8-card DeepSeek-V4-Flash service -set -e - -cd /data/user1/yy -mkdir -p logs - -export PYTHONUNBUFFERED=1 -export SGLANG_LOG_LEVEL=info -export PATH="/data/user1/yy/envs/sglang/bin:$PATH" - -LOG_FILE="logs/sglang_8card_$(date +%Y%m%d_%H%M%S).log" - -echo "Starting sglang 8-card DSV4 service, logging to ${LOG_FILE}" - -exec /data/user1/yy/envs/sglang/bin/sglang serve \ - --trust-remote-code \ - --model-path /data/models/DeepSeek-V4-Flash \ - --tp 8 \ - --moe-runner-backend marlin \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --host 0.0.0.0 \ - --port 30000 \ - 2>&1 | tee "${LOG_FILE}" diff --git a/scripts/start_vllm_eagle_dsv4_8card.sh b/scripts/start_vllm_eagle_dsv4_8card.sh deleted file mode 100644 index 676998e..0000000 --- a/scripts/start_vllm_eagle_dsv4_8card.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-main-latest" -PYTHON="$VENV/bin/python" -VLLM="$VENV/bin/vllm" -export PATH="$VENV/bin:$PATH" - -MODEL="/data/models/DeepSeek-V4-Flash" -PORT=30005 -TP=8 -LOG="/data/user1/yy/logs/vllm_eagle_dsv4_tp${TP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/vllm_eagle_dsv4.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash vLLM EAGLE (TP=$TP) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" -nohup "$VLLM" serve "$MODEL" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 256 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --spec-method mtp \ - --spec-model "$MODEL" \ - --spec-tokens 4 \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port "$PORT" \ - > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -200 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -200 "$LOG" -exit 1 diff --git a/scripts/start_vllm_mtp_dsv4_8card.sh b/scripts/start_vllm_mtp_dsv4_8card.sh deleted file mode 100644 index 1c6a20c..0000000 --- a/scripts/start_vllm_mtp_dsv4_8card.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -set -e - -cd /data/user1/yy -mkdir -p logs - -VENV="/data/user1/yy/envs/vllm-dspark" -export PATH="$VENV/bin:$PATH" -PYTHON="$VENV/bin/python" -VLLM="$VENV/bin/vllm" - -MODEL="/data/models/DeepSeek-V4-Flash" -PORT=30005 -TP=8 -LOG="/data/user1/yy/logs/vllm_mtp_dsv4_tp${TP}_$(date +%Y%m%d_%H%M%S).log" -PID_FILE="/data/user1/yy/vllm_mtp_dsv4.pid" - -export TMPDIR=/data/user1/yy/tmp -export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 - -echo "=== Starting DeepSeek-V4-Flash vLLM MTP (TP=$TP) ===" -echo "Model: $MODEL" -echo "Port: $PORT" -echo "Log: $LOG" - -rm -f "$PID_FILE" -nohup "$VLLM" serve "$MODEL" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 128 \ - --gpu-memory-utilization 0.85 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --spec-method mtp \ - --spec-model "$MODEL" \ - --spec-tokens 4 \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port "$PORT" \ - > "$LOG" 2>&1 & -PID=$! -echo $PID > "$PID_FILE" -echo "PID: $PID" -echo "Waiting for health..." - -for i in $(seq 1 240); do - if curl -s "http://127.0.0.1:$PORT/health" > /dev/null 2>&1; then - echo "Server is ready at http://127.0.0.1:$PORT" - echo "Log: $LOG" - exit 0 - fi - if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Server exited early" - tail -200 "$LOG" - exit 1 - fi - echo "Waiting... ($i/240)" - sleep 5 -done - -echo "ERROR: Server not healthy after 240 retries" -tail -200 "$LOG" -exit 1 diff --git a/scripts/start_vllm_pd_single_node.sh b/scripts/start_vllm_pd_single_node.sh deleted file mode 100755 index 667282d..0000000 --- a/scripts/start_vllm_pd_single_node.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -set -e - -# vLLM + Mooncake PD 分离单节点启动脚本 -# 104 上前 4 张卡做 prefill，后 4 张卡做 decode -# 使用 TCP 传输（因为当前 IB/RDMA 未启用） - -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -BOOTSTRAP_PORT=8998 -PREFILL_PORT=8000 -DECODE_PORT=8001 -ROUTER_PORT=30000 -HOST="0.0.0.0" - -VENV="/data/user1/yy/envs/vllm" -PYTHON="$VENV/bin/python" -VLLM="$VENV/bin/vllm" -VLLM_ROUTER="$VENV/bin/vllm-router" -LOG_DIR="/data/user1/yy/vllm_pd_logs" -mkdir -p "$LOG_DIR" - -echo "=== Starting vLLM PD disaggregation (single node) ===" -echo "Model: $MODEL_PATH" -echo "Prefill GPUs: 0-3, port $PREFILL_PORT" -echo "Decode GPUs: 4-7, port $DECODE_PORT" -echo "Router port: $ROUTER_PORT" -echo "Logs: $LOG_DIR" -echo "" - -# Clean up old logs -rm -f "$LOG_DIR"/*.log - -# 使用 TCP 协议（IB 没起来，不能用 rdma） -export VLLM_MOONCAKE_PROTOCOL=tcp -export VLLM_MOONCAKE_BOOTSTRAP_PORT=$BOOTSTRAP_PORT - -# Common args -PREFILL_KV_CONFIG='{"kv_connector":"MooncakeConnector","kv_role":"kv_producer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"mooncake_protocol":"tcp"}}' -DECODE_KV_CONFIG='{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"mooncake_protocol":"tcp"}}' - -# Start prefill server (GPUs 0-3) -echo "[1/3] Starting prefill server on GPUs 0-3..." -CUDA_VISIBLE_DEVICES=0,1,2,3 nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --port "$PREFILL_PORT" \ - --data-parallel-size 4 \ - --enable-expert-parallel \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --max-model-len auto \ - --max-num-batched-tokens 16384 \ - --max-num-seqs 8 \ - --enforce-eager \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --kv-transfer-config "$PREFILL_KV_CONFIG" \ - > "$LOG_DIR/prefill.log" 2>&1 & -PREFILL_PID=$! -echo "Prefill PID: $PREFILL_PID" - -# Wait for prefill health -for i in {1..120}; do - if curl -s "http://127.0.0.1:$PREFILL_PORT/health" > /dev/null 2>&1; then - echo "Prefill server is ready" - break - fi - if ! kill -0 $PREFILL_PID 2>/dev/null; then - echo "ERROR: Prefill server exited early" - tail -80 "$LOG_DIR/prefill.log" - exit 1 - fi - echo "Waiting for prefill server... ($i/120)" - sleep 5 -done - -# Start decode server (GPUs 4-7) -echo "[2/3] Starting decode server on GPUs 4-7..." -CUDA_VISIBLE_DEVICES=4,5,6,7 nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --port "$DECODE_PORT" \ - --data-parallel-size 4 \ - --enable-expert-parallel \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --max-model-len auto \ - --max-num-seqs 512 \ - --max-num-batched-tokens 512 \ - --compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY","max_cudagraph_capture_size":512,"compile_ranges_endpoints":[512]}' \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --kv-transfer-config "$DECODE_KV_CONFIG" \ - > "$LOG_DIR/decode.log" 2>&1 & -DECODE_PID=$! -echo "Decode PID: $DECODE_PID" - -# Wait for decode health -for i in {1..120}; do - if curl -s "http://127.0.0.1:$DECODE_PORT/health" > /dev/null 2>&1; then - echo "Decode server is ready" - break - fi - if ! kill -0 $DECODE_PID 2>/dev/null; then - echo "ERROR: Decode server exited early" - tail -80 "$LOG_DIR/decode.log" - exit 1 - fi - echo "Waiting for decode server... ($i/120)" - sleep 5 -done - -# Start vllm-router -echo "[3/3] Starting vllm-router on port $ROUTER_PORT..." -nohup "$VLLM_ROUTER" \ - --policy round_robin \ - --vllm-pd-disaggregation \ - --prefill "http://127.0.0.1:$PREFILL_PORT" \ - --decode "http://127.0.0.1:$DECODE_PORT" \ - --host 127.0.0.1 \ - --port "$ROUTER_PORT" \ - --intra-node-data-parallel-size 4 \ - --kv-connector mooncake \ - > "$LOG_DIR/router.log" 2>&1 & -ROUTER_PID=$! -echo "Router PID: $ROUTER_PID" - -# Wait for router health -for i in {1..60}; do - if curl -s "http://127.0.0.1:$ROUTER_PORT/health" > /dev/null 2>&1; then - echo "Router is ready" - break - fi - if ! kill -0 $ROUTER_PID 2>/dev/null; then - echo "ERROR: Router exited early" - tail -50 "$LOG_DIR/router.log" - exit 1 - fi - echo "Waiting for router... ($i/60)" - sleep 2 -done - -echo "" -echo "=== vLLM PD disaggregation is ready ===" -echo "Router URL: http://127.0.0.1:$ROUTER_PORT" -echo "Prefill URL: http://127.0.0.1:$PREFILL_PORT" -echo "Decode URL: http://127.0.0.1:$DECODE_PORT" -echo "" -echo "Test command:" -echo " curl http://127.0.0.1:$ROUTER_PORT/v1/completions \\" -echo " -H \"Content-Type: application/json\" \\" -echo " -d '{\"model\":\"DeepSeek-V4-Flash\",\"prompt\":\"Hello\",\"max_tokens\":32}'" -echo "" -echo "To stop: kill $PREFILL_PID $DECODE_PID $ROUTER_PID" diff --git a/scripts/wait_and_profile_dspark.sh b/scripts/wait_and_profile_dspark.sh deleted file mode 100755 index 944b5c2..0000000 --- a/scripts/wait_and_profile_dspark.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# Wait for GPUs to become idle (no other compute processes), then run DSpark profiles. - -set -e - -ROOT=/data/user1/yy -PROFILE_PID_FILE=/tmp/profile_dspark.pid -MONITOR_LOG=$ROOT/bench_results/dspark_profile_monitor.log -RESULT_BASE=$ROOT/bench_results/dspark_profile_$(date +%Y%m%d_%H%M%S) - -mkdir -p "$RESULT_BASE" "$ROOT/tmp" - -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$MONITOR_LOG" -} - -is_gpu_idle() { - # Consider idle if no process other than ours uses > 1GB on any GPU. - # Returns 0 (true) if idle, 1 (false) if busy. - nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits | \ - awk -F',' ' - { - pid=$1; mem=$2; - # Exclude our own known PIDs (monitor script, shell, etc.) - if (mem > 1024) { busy++; } - } - END { exit (busy > 0 ? 1 : 0) } - ' -} - -log "Starting GPU idle monitor. Will check every 5 minutes." -log "Results will go to $RESULT_BASE" - -wait_for_idle() { - while true; do - if is_gpu_idle; then - log "GPUs appear idle." - return 0 - else - log "GPUs still busy. Sleeping 5 minutes." - sleep 300 - fi - done -} - -# Run profiles sequentially, checking GPU idle state before each run. -CONFIGS=("dspark-st3:64" "dspark-st5:64" "dspark-st3:1" "nospec:64") - -for entry in "${CONFIGS[@]}"; do - IFS=':' read -r config concurrency <<< "$entry" - log "Waiting before profile: config=$config concurrency=$concurrency" - wait_for_idle - log "Running profile: config=$config concurrency=$concurrency" - "$ROOT/envs/vllm-dspark/bin/python" "$ROOT/scripts/profile_dspark.py" \ - --config "$config" --concurrency "$concurrency" --tool nsys \ - >> "$RESULT_BASE/profile_${config}_c${concurrency}.log" 2>&1 - log "Finished profile: config=$config concurrency=$concurrency" -done - -log "All profiles complete. Results in $RESULT_BASE"