diff --git a/.gitignore b/.gitignore index 0b81dda..c4c4105 100644 --- a/.gitignore +++ b/.gitignore @@ -24,8 +24,8 @@ datasets/ bench_results/**/raw_outputs/ # 实验级原始结果目录：保留 report.md / results.json，忽略原始 jsonl 和日志 -experiments/*/results/raw_outputs/ -experiments/*/results/logs/ +experiments/*/results/*/raw_outputs/ +experiments/*/results/*/logs/ # 无关项目 loomeval_yy/ diff --git a/README.md b/README.md index e7c5804..84fb8b4 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ | DSV4 P800 SGLang | `experiments/dsv4_p800_sglang/run_bench.sh` | Kunlun P800 + SGLang + DeepSeek-V4-Flash-INT8 | | DSV4 H200 DSpark | `experiments/dsv4_h200_dspark/run_bench.sh` | NVIDIA H200 + vllm-dspark + DeepSeek-V4-Flash-DSpark | | DSV4 H200 vLLM baseline | `experiments/dsv4_h200_vllm/run_bench.sh` | NVIDIA H200 + vLLM + DeepSeek-V4-Flash | +| DSV4 H200 SGLang baseline | `experiments/dsv4_h200_sglang/run_bench.sh` | NVIDIA H200 + native SGLang + DeepSeek-V4-Flash | ### 旧结构（scripts/ + bench_results/） @@ -55,6 +56,12 @@ bash experiments/dsv4_p800_sglang/run_bench.sh bash experiments/dsv4_h200_dspark/run_bench.sh ``` +### H200 SGLang baseline + +```bash +bash experiments/dsv4_h200_sglang/run_bench.sh +``` + ### DSpark grid（旧结构） ```bash diff --git a/experiments/dsv4_h200_dspark/DSPARK_FIX_PR_PREP.md b/experiments/dsv4_h200_dspark/DSPARK_FIX_PR_PREP.md deleted file mode 100644 index 25c4071..0000000 --- a/experiments/dsv4_h200_dspark/DSPARK_FIX_PR_PREP.md +++ /dev/null @@ -1,286 +0,0 @@ -# DeepSeek-V4-Flash-DSpark 修复记录（GitHub Issue #47648） - -> 本文件记录 vLLM DSpark 在 DeepSeek-V4-Flash 上的启动失败问题、根因、修复 diff 以及验证结果，便于后续直接用于提交 PR。 - ---- - -## 1. 问题概述 - -在 H200/SM90（以及同样走 NVIDIA 路径的 B200/SM120）上，使用 `--spec-method dspark` 部署 `DeepSeek-V4-Flash-DSpark` 时，模型初始化阶段会失败。主要表现为两类错误： - -1. **DSpark draft 权重加载路径错配**：draft 权重（checkpoint 中以 `mtp.{i}.*` 命名）没有被正确加载到模型中。 -2. **KV cache shape mismatch**：当启用 `fp8_ds_mla` layout 时，KV cache 的 per-token slot size 被错误计算为 512B，而实际需要 584B，导致 shape 断言失败。 - -关联 issue：[vllm-project/vllm#47648](https://github.com/vllm-project/vllm/issues/47648) - ---- - -## 2. 环境信息 - -- vLLM 版本：`0.23.1rc1.dev788+gfa4321de3`（vllm-dspark wheel） -- PyTorch：`2.11.0+cu129` -- GPU：8× NVIDIA H200（SM90），TP=8 -- 模型：`/data/models/DeepSeek-V4-Flash-DSpark` -- 启动命令： - -```bash -vllm serve /data/models/DeepSeek-V4-Flash-DSpark \ - --trust-remote-code \ - --tensor-parallel-size 8 \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --max-model-len auto \ - --max-num-seqs 256 \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --spec-method dspark \ - --spec-model /data/models/DeepSeek-V4-Flash-DSpark \ - --spec-tokens 5 \ - --no-disable-hybrid-kv-cache-manager \ - --disable-uvicorn-access-log \ - --port 30004 -``` - ---- - -## 3. 根因分析 - -### 3.1 根因一：DSpark draft 权重加载路径错配 - -#### 位置 - -`vllm/models/deepseek_v4/nvidia/dspark.py`，`_remap_dspark_name` 方法。 - -#### 问题描述 - -DeepSeek-V4-Flash-DSpark 的 draft 权重是嵌在目标模型 checkpoint 中的，命名格式为 `mtp.{i}.*`，例如： - -```text -mtp.0.self_attn.wq_b.weight -mtp.0.ffn.experts.0.w1.weight -mtp.1.self_attn.wq_b.weight -... -``` - -`_remap_dspark_name` 负责把这些 checkpoint key 映射到 DSpark draft 模型内部的真实参数名。原来的实现： - -```python -return f"model.layers.{self.num_hidden_layers + stage}.{rest}" -``` - -把 `mtp.0.*` 映射成了 `model.layers.{num_hidden_layers + 0}.*`（例如 `model.layers.64.*`）。 - -但实际上，DSpark draft 的 3 个 `DeepseekV4DecoderLayer` 是放在一个 `nn.ModuleList` 里的，PyTorch 真实注册的参数名只跟 `ModuleList` 的索引有关： - -```text -model.layers.0.* -model.layers.1.* -model.layers.2.* -``` - -`DeepseekV4DecoderLayer` 构造函数里传入的 `prefix=f"layers.{num_hidden_layers + i}"` 只是为了内部计算 `compress_ratio`（让 `layer_id >= num_hidden_layers` 时固定 `compress_ratio=1`），**不是真实的参数名前缀**。 - -因此，原来的映射导致所有 draft block 权重都找不到对应的模型参数，draft layer 实际上没有被正确加载。 - -#### 修复 - -把 `mtp.{i}` 的 block 权重映射到 `model.layers.{i}`： - -```python -def _remap_dspark_name(self, name: str) -> str | None: - """Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path. - - Returns None for non-mtp weights (owned by the target model). - """ - m = re.match(r"mtp\.(\d+)\.(.*)", name) - if m is None: - return None - stage = int(m.group(1)) - rest = m.group(2) - # The confidence head is not wired into inference yet; drop its weights. - if rest.startswith("confidence_head."): - return None - # Head-stack params live at model level (mtp.last), context combiner at - # model level (mtp.0); everything else is a per-layer decoder block. - head_prefixes = ( - "norm.", - "hc_head_fn", - "hc_head_base", - "hc_head_scale", - "markov_head.", - ) - if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith( - head_prefixes - ): - return f"model.{rest}" - # Draft layers live in a ModuleList, so their actual parameter names are - # model.layers.{stage}.* even though the prefix passed to the decoder - # layer is layers.{num_hidden_layers + stage} (for compress_ratio). - return f"model.layers.{stage}.{rest}" -``` - -### 3.2 根因二：DeepSeek-V4 KV cache shape mismatch - -#### 位置 - -`vllm/models/deepseek_v4/attention.py`，`DeepseekV4Attention.get_kv_cache_spec` 方法。 - -#### 问题描述 - -`DeepseekV4Attention.get_kv_cache_spec` 返回的 `MLAAttentionSpec` 没有设置 `kv_quant_mode`。在 `gpu_model_runner.py` 中，这个缺失导致 `cache_dtype_str` 被当成 `"auto"` 传给 `DeepseekV4FlashMLABackend.get_kv_cache_shape`，返回的 shape 是 `(num_blocks, block_size, 512)`。 - -但对于 `fp8_ds_mla` layout（UE8M0 block-scaled fp8，以 `uint8` 打包），每个 token 的 KV slot 实际大小是 584 字节，而不是 512。于是 KV cache 分配的空间不够，触发 shape mismatch / assert。 - -#### 修复 - -在 `MLAAttentionSpec` 中加上 `kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype)`： - -```python -def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: - if ( - self.compress_ratio <= 1 - ): # SWA part. Allocated separately as DeepseekV4SWACache. - return None - # fp8_ds_mla is a UE8M0 block-scaled uint8 layout and needs 576B - # alignment; plain bf16 / per-tensor fp8 rows use natural element-size - # pages. - uses_fp8_ds_mla_layout = self.kv_cache_dtype == "fp8_ds_mla" - return MLAAttentionSpec( - block_size=vllm_config.cache_config.block_size, - num_kv_heads=1, - head_size=self.head_dim, - dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype, - compress_ratio=self.compress_ratio, - cache_dtype_str=self.kv_cache_dtype, - alignment=576 if uses_fp8_ds_mla_layout else None, - model_version="deepseek_v4", - kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), - ) -``` - ---- - -## 4. 修改文件清单 - -| 文件 | 修改内容 | -|---|---| -| `vllm/models/deepseek_v4/nvidia/dspark.py` | `_remap_dspark_name`：draft block 权重从 `model.layers.{num_hidden_layers + stage}` 改为 `model.layers.{stage}` | -| `vllm/models/deepseek_v4/attention.py` | `DeepseekV4Attention.get_kv_cache_spec`：在 `MLAAttentionSpec` 中补充 `kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype)` | - -> 如果上游 vllm-main 仓库与 vllm-dspark 安装包的文件结构一致，也需要同步修改 `vllm-main` 下对应路径的同名文件。 - ---- - -## 5. 完整 diff（面向 PR） - -```diff ---- a/vllm/models/deepseek_v4/nvidia/dspark.py -+++ b/vllm/models/deepseek_v4/nvidia/dspark.py -@@ -486,8 +486,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): - if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith( - head_prefixes - ): - return f"model.{rest}" -- # Draft layers live after the target layers in the decoder stack. -- return f"model.layers.{self.num_hidden_layers + stage}.{rest}" -+ # Draft layers live in a ModuleList, so their actual parameter names are -+ # model.layers.{stage}.* even though the prefix passed to the decoder -+ # layer is layers.{num_hidden_layers + stage} (for compress_ratio). -+ return f"model.layers.{stage}.{rest}" -``` - -```diff ---- a/vllm/models/deepseek_v4/attention.py -+++ b/vllm/models/deepseek_v4/attention.py -@@ -613,6 +613,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): - cache_dtype_str=self.kv_cache_dtype, - alignment=576 if uses_fp8_ds_mla_layout else None, - model_version="deepseek_v4", -+ kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), - ) -``` - ---- - -## 6. 验证结果 - -### 6.1 Qwen3 DSpark 通路验证 - -- 目标模型：`/data/models/Qwen3-4B` -- Draft 模型：`deepseek-ai/dspark_qwen3_4b_block7` -- 结果：服务启动成功，`/v1/completions` 返回结果正常，speculative decoding 工作。 - -### 6.2 DeepSeek-V4-Flash-DSpark 服务验证 - -修复后，使用第 2 节的命令可以成功启动服务，监听 `http://127.0.0.1:30004`。简单 completion 请求返回正常： - -```bash -curl -s -X POST http://127.0.0.1:30004/v1/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "/data/models/DeepSeek-V4-Flash-DSpark", - "prompt": "The capital of France is", - "max_tokens": 20, - "temperature": 0.0 - }' -# 返回: " Paris." -``` - -### 6.3 Serving benchmark - -使用 vLLM 内置 `vllm bench serve`： - -```bash -vllm bench serve \ - --host 127.0.0.1 --port 30004 \ - --backend openai \ - --dataset-name sharegpt \ - --dataset-path /data/user1/yy/datasets/ShareGPT_filtered_chat.json \ - --sharegpt-output-len 256 \ - --num-prompts 50 \ - --max-concurrency 16 \ - --endpoint /v1/completions \ - --model /data/models/DeepSeek-V4-Flash-DSpark \ - --seed 42 -``` - -结果： - -| 指标 | 数值 | -|---|---| -| Request throughput | **2.38 req/s** | -| Output token throughput | **610.47 tok/s** | -| Acceptance rate | **28.20%** | -| Mean acceptance length | **2.41** | -| Mean TTFT | 2162 ms | -| Mean TPOT | 16.85 ms | -| Total input tokens | 16381 | -| Total generated tokens | 12800 | - -服务器日志中的 SpecDecoding metrics 在稳态下 acceptance rate 落在 25%–36% 区间，与 DSpark 论文预期一致。 - ---- - -## 7. 影响范围 - -- **受影响**：所有使用 NVIDIA 路径运行 DeepSeek-V4-Flash-DSpark 的 GPU（包括 H200/SM90 的 FlashMLA 路径，以及 B200/SM120 的 FlashInfer SM120 路径）。 -- **不受影响**： - - Qwen3 DSpark（使用独立的 `qwen3_dspark.py` 实现，不共享 `_remap_dspark_name`）。 - - AMD/ROCm 和 XPU 平台（目前 DSpark 仅支持 NVIDIA）。 - ---- - -## 8. 后续 PR 待办 - -- [ ] 在 vllm-main 上同步应用以上两处修改。 -- [ ] 跑通 DeepSeek-V4-Flash-DSpark 的单元测试 / 冒烟测试。 -- [ ] 检查 `DeepseekV4IndexerCache.get_kv_cache_spec` 是否也需要补充 `kv_quant_mode`（当前未改动，因为未触发错误）。 -- [ ] 补充 DSpark draft 权重加载的回归测试（可选，但建议）。 -- [ ] 向 vllm-project/vllm 提交 PR，并在描述中引用 issue #47648。 - ---- - -## 9. 备注 - -- 本文件中的 token、路径、版本号均基于 2026-07-05 的实际运行环境。 -- GitHub issue 评论已发布：https://github.com/vllm-project/vllm/issues/47648#issuecomment-4886275279 diff --git a/experiments/dsv4_h200_sglang/README.md b/experiments/dsv4_h200_sglang/README.md new file mode 100644 index 0000000..43ee5dd --- /dev/null +++ b/experiments/dsv4_h200_sglang/README.md @@ -0,0 +1,42 @@ +# DSV4 H200 SGLang Baseline Benchmark + +NVIDIA H200 + native `sglang` + `DeepSeek-V4-Flash` baseline benchmark experiment. + +This experiment uses: +- `/data/user1/yy/envs/sglang` as both the server and benchmark client environment + +## Quick Start + +```bash +# Run the full experiment (start server + benchmark + parse) +bash experiments/dsv4_h200_sglang/run_bench.sh + +# Reuse an already-running server +SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_sglang/run_bench.sh +``` + +Results land in `experiments/dsv4_h200_sglang/results//`. + +## Configuration + +Edit `config.env` or override via environment variables: + +```bash +MODEL_PATH=/data/models/DeepSeek-V4-Flash \ +PORT=30006 \ +SCENARIOS=("32 512 256" "128 512 256") \ +bash experiments/dsv4_h200_sglang/run_bench.sh +``` + +## Files + +| File | Purpose | +|---|---| +| `config.env` | Experiment-level configuration (model, port, venv paths, scenarios) | +| `start_server.sh` | Start a plain SGLang baseline server for DeepSeek-V4-Flash | +| `run_bench.sh` | Orchestrator: metadata → server → benchmark → parse | +| `parse_results.py` | Parse raw JSONL outputs into `results.json` + `report.md` | + +## Platform + +This experiment targets `platforms/nvidia_h200.env`. diff --git a/experiments/dsv4_h200_sglang/config.env b/experiments/dsv4_h200_sglang/config.env new file mode 100644 index 0000000..2fc51bb --- /dev/null +++ b/experiments/dsv4_h200_sglang/config.env @@ -0,0 +1,28 @@ +# Experiment-level configuration for dsv4_h200_sglang +# All values can be overridden via environment variables. + +EXPERIMENT="${EXPERIMENT:-dsv4_h200_sglang}" +MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash}" +MODEL_PATH="${MODEL_PATH:-/data/models/DeepSeek-V4-Flash}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash}" +PORT="${PORT:-30006}" +# This experiment is a plain SGLang baseline. These values override the +# vllm-dspark defaults coming from platforms/nvidia_h200.env. +BACKEND="sglang" +ENGINE="sglang" + +# Native virtual environments on the host. +VENV_SERVER="${VENV_SERVER:-/data/user1/yy/envs/sglang}" +VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}" + +# Benchmark scenarios: each element is "concurrency input_len output_len". +if [[ -z "${SCENARIOS:-}" ]]; then + SCENARIOS=( + "32 512 256" + "128 512 256" + ) +fi +NUM_PROMPTS="${NUM_PROMPTS:-128}" + +# Server start script bundled with this experiment. +SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_sglang/parse_results.py b/experiments/dsv4_h200_sglang/parse_results.py new file mode 100755 index 0000000..0ac603b --- /dev/null +++ b/experiments/dsv4_h200_sglang/parse_results.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Parse H200 SGLang baseline benchmark JSONL outputs. + +Reads the single-line summary JSON produced by +`sglang.bench_serving --output-file --output-details` and generates: + - results.json (appended scenarios) + - report.md (human-readable summary) + +Usage: + python3 parse_results.py +""" + +import json +import sys +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + """Read the first (and usually only) JSON object from the JSONL file.""" + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute_metrics(data: dict) -> dict: + completed = data.get("completed", 0) + total = len(data.get("input_lens", [])) + failed = total - completed if total > 0 else 0 + duration_s = data.get("duration", 0.0) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_token_throughput": data.get("total_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def append_scenario(results_json: Path, scenario: dict) -> None: + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"].append(scenario) + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def generate_report(result_root: Path, scenarios: list[dict]) -> None: + report_path = result_root / "report.md" + with open(report_path, "w", encoding="utf-8") as f: + f.write("# H200 SGLang Baseline Benchmark Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n") + f.write("- Backend: SGLang (TP=8, moe-runner-backend=marlin, no speculative decoding)\n") + f.write("- Benchmark client: `sglang.bench_serving --backend sglang`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + f.write( + f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} |\n" + ) + f.write("\n") + + +def main() -> None: + result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob("sglang_*.jsonl")): + # Filename: sglang_MMDD_concurrency_inputlen_outputlen.jsonl + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) + + data = parse_jsonl(jsonl_path) + if data is None: + continue + + metrics = compute_metrics(data) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": "random", + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + for s in scenarios: + append_scenario(results_json, s) + + generate_report(result_root, scenarios) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_sglang/results/20260708-065101/report.md b/experiments/dsv4_h200_sglang/results/20260708-065101/report.md new file mode 100644 index 0000000..9a03540 --- /dev/null +++ b/experiments/dsv4_h200_sglang/results/20260708-065101/report.md @@ -0,0 +1,14 @@ +# H200 SGLang Baseline Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_sglang/results/20260708-065101` +- Model: `/data/models/DeepSeek-V4-Flash` +- Backend: SGLang (TP=8, moe-runner-backend=marlin, no speculative decoding) +- Benchmark client: `sglang.bench_serving --backend sglang` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i512_o256 | 128 | 512 | 256 | 5.19 | 128 | 24.65 | 6787.39 | 3254.28 | 10041.66 | 924.15 | 1207.75 | 1210.04 | 26.29 | 46.30 | 145.35 | 3471.49 | 5056.32 | 5112.01 | +| c32_i512_o256 | 32 | 512 | 256 | 22.32 | 128 | 5.74 | 1579.38 | 757.25 | 2336.63 | 380.29 | 766.65 | 772.20 | 36.82 | 48.50 | 60.81 | 5134.63 | 10278.13 | 10792.89 | + diff --git a/experiments/dsv4_h200_sglang/results/20260708-065101/results.json b/experiments/dsv4_h200_sglang/results/20260708-065101/results.json new file mode 100644 index 0000000..7e6f75b --- /dev/null +++ b/experiments/dsv4_h200_sglang/results/20260708-065101/results.json @@ -0,0 +1,130 @@ +{ + "metadata": { + "experiment": "dsv4_h200_sglang", + "run_id": "20260708-065101", + "timestamp": "2026-07-08T06:51:01+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "sglang", + "engine": "sglang", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_sglang/run_bench.sh", + "env": "/data/user1/yy/envs/sglang", + "git_commit": "7af3d4d", + "git_dirty": "dirty", + "description": "H200 native SGLang baseline benchmark for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "moe_runner_backend": "marlin", + "port": 30006, + "num_prompts": 128, + "scenarios": [ + "32 512 256", + "128 512 256" + ] + }, + "scenarios": [ + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 5.193163285992341, + "request_throughput": 24.647790364161636, + "input_token_throughput": 6787.38527153101, + "output_token_throughput": 3254.278571518216, + "total_token_throughput": 10041.663843049226, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 3471.4865707105673, + "p50": 3514.0166565033724, + "p90": 4998.641918299836, + "p95": 5056.324887800292, + "p99": 5112.00928252787 + }, + "ttft_ms": { + "mean": 924.1514429138533, + "p50": 849.3422594983713, + "p90": 1205.6520546015236, + "p95": 1207.7475340476667, + "p99": 1210.0433263290324 + }, + "tpot_ms": { + "mean": 26.29247494673112, + "p50": 19.806734621095913, + "p90": 35.49035216999157, + "p95": 46.297342646573306, + "p99": 145.35319635262547 + }, + "itl_ms": { + "mean": 19.44163585265753, + "p50": 15.339888006565161, + "p90": 17.66490399313625, + "p95": 18.918996000138577, + "p99": 153.02511929621687 + } + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang/results/20260708-065101/raw_outputs/sglang_0708_128_512_256.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 128 + }, + "metrics": { + "success": 128, + "failed": 0, + "duration_s": 22.31761664900114, + "request_throughput": 5.73537945440643, + "input_token_throughput": 1579.3801172571705, + "output_token_throughput": 757.2493185895989, + "total_token_throughput": 2336.6294358467694, + "total_input_tokens": 35248, + "total_output_tokens": 16900, + "e2e_ms": { + "mean": 5134.632653607923, + "p50": 4641.428427006758, + "p90": 9806.394101392652, + "p95": 10278.134901099836, + "p99": 10792.887551254826 + }, + "ttft_ms": { + "mean": 380.2895297493478, + "p50": 331.6603470011614, + "p90": 642.7646050986368, + "p95": 766.6534427480656, + "p99": 772.1964311384363 + }, + "tpot_ms": { + "mean": 36.81840456313315, + "p50": 38.839227149440994, + "p90": 44.66492350042031, + "p95": 48.50295195923065, + "p99": 60.81439458782757 + }, + "itl_ms": { + "mean": 36.28599313285003, + "p50": 11.074887996073812, + "p90": 154.03797599719837, + "p95": 164.296265997109, + "p99": 203.1443601998035 + } + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_sglang/results/20260708-065101/raw_outputs/sglang_0708_32_512_256.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_sglang/run_bench.sh b/experiments/dsv4_h200_sglang/run_bench.sh new file mode 100755 index 0000000..18b4044 --- /dev/null +++ b/experiments/dsv4_h200_sglang/run_bench.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# H200 native SGLang baseline benchmark for DeepSeek-V4-Flash. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" +RAW_DIR="${RESULT_ROOT}/raw_outputs" +LOG_DIR="${RESULT_ROOT}/logs" + +ensure_result_root "$RESULT_ROOT" +log_init "${LOG_DIR}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "result_root=${RESULT_ROOT}" +log "platform=${PLATFORM}" +log "chip=${CHIP}" +log "accelerator=${ACCELERATOR}" +log "engine=${ENGINE}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "server_start_script=${SERVER_START_SCRIPT}" + +# Write initial metadata for this run. +METADATA_JSON="${RESULT_ROOT}/results.json" +write_metadata_json \ + "$METADATA_JSON" \ + "$EXPERIMENT_NAME" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$VENV_SERVER" \ + "H200 native SGLang baseline benchmark for DeepSeek-V4-Flash" + +# Update metadata with config. +"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' +import json +import sys + +path = sys.argv[1] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "moe_runner_backend": "marlin", + "port": 30006, + "num_prompts": 128, + "scenarios": ["32 512 256", "128 512 256"] +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local pid_file="/data/user1/yy/dsv4_h200_sglang.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping existing sglang server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true + sleep 2 +} + +start_server() { + log "starting sglang server with ${SERVER_START_SCRIPT}" + if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then + log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" + exit 1 + fi + bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 + if ! is_server_healthy; then + log "error: sglang server failed to become healthy" + exit 1 + fi + log "sglang server is healthy" +} + +on_exit() { + local code=$? + log "orchestrator exiting with code=${code}" + if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then + stop_server + fi + exit "$code" +} +trap on_exit EXIT + +if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then + log "SKIP_MANAGE_SERVER is set, assuming server is already running on port ${PORT}" + if ! is_server_healthy; then + log "error: no healthy server found at port ${PORT}" + exit 1 + fi +else + stop_server + start_server +fi + +log "===== BENCHMARK START =====" + +for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len <<< "$scenario" + output_file="${RAW_DIR}/sglang_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${LOG_DIR}/sglang_c${concurrency}_i${input_len}_o${output_len}.log" + + log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "$PORT" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts "$NUM_PROMPTS" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished scenario: output=${output_file}" +done + +log "===== BENCHMARK DONE =====" + +log "parsing results" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { + log "WARNING: parser failed; see ${LOG_DIR}/parse.log" +} + +log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_sglang/start_server.sh b/experiments/dsv4_h200_sglang/start_server.sh new file mode 100755 index 0000000..fb274e0 --- /dev/null +++ b/experiments/dsv4_h200_sglang/start_server.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Start a SGLang baseline server for DeepSeek-V4-Flash on H200. +set -e + +cd /data/user1/yy +mkdir -p logs + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +export PATH="${VENV_SERVER}/bin:$PATH" +export PYTHONUNBUFFERED=1 +export SGLANG_LOG_LEVEL=info + +TP=8 +LOG="/data/user1/yy/logs/dsv4_h200_sglang_tp${TP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_sglang.pid" + +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 + +echo "=== Starting DeepSeek-V4-Flash SGLang baseline (TP=$TP) ===" +echo "Model: $MODEL_PATH" +echo "Port: $PORT" +echo "Log: $LOG" + +rm -f "$PID_FILE" +nohup "${VENV_SERVER}/bin/sglang" serve \ + --trust-remote-code \ + --model-path "$MODEL_PATH" \ + --tp "$TP" \ + --moe-runner-backend marlin \ + --host 0.0.0.0 \ + --port "$PORT" \ + > "$LOG" 2>&1 & +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + # curl --fail treats any HTTP >=400 as an error, so 503 won't pass. + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then + echo "Server is ready at http://127.0.0.1:$PORT" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Server not healthy after 240 retries" +tail -200 "$LOG" +exit 1