feat(256k-4k-probe): verify SGLang/vLLM can serve 256k+4k without OOM

2026-07-08 09:59:05 +00:00 · 2026-07-08 09:59:05 +00:00 · cd2ba29d49
commit cd2ba29d49
parent 0f9399baf6
12 changed files with 1033 additions and 0 deletions
--- a/experiments/dsv4_h200_256k_4k_probe/compare.py
+++ b/experiments/dsv4_h200_256k_4k_probe/compare.py
@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Generate a side-by-side comparison of SGLang and vLLM results.
+
+Usage:
+    python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
+        [--output comparison.md]
+"""
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+
+
+def load_result(result_root: Path) -> dict:
+    path = result_root / "results.json"
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
+    # S2 tier targets from scripts/SLO_STANDARDS.md.
+    ttft_ok = ttft_p95_ms < 3000.0
+    tpot_ok = tpot_mean_ms < 50.0
+    if ttft_ok and tpot_ok:
+        return "✅"
+    if ttft_ok or tpot_ok:
+        return "⚠️"
+    return "❌"
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--sglang", type=Path, required=True)
+    parser.add_argument("--vllm", type=Path, required=True)
+    parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
+    args = parser.parse_args()
+
+    sglang_data = load_result(args.sglang)
+    vllm_data = load_result(args.vllm)
+
+    by_scenario = defaultdict(dict)
+    for data in (sglang_data, vllm_data):
+        backend = data["metadata"]["engine"]
+        for s in data.get("scenarios", []):
+            key = s["name"]
+            by_scenario[key][backend] = s
+
+    with open(args.output, "w", encoding="utf-8") as f:
+        f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
+        f.write("## Summary\n\n")
+        f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
+        f.write("- Hardware: 8x NVIDIA H200 143GB\n")
+        f.write("- Tensor Parallelism: 8\n")
+        f.write("- Benchmark client: `sglang.bench_serving`\n")
+        f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
+
+        f.write("## Side-by-side results\n\n")
+        f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
+        f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
+
+        for scenario_name in sorted(by_scenario.keys()):
+            for backend in ("sglang", "vllm"):
+                s = by_scenario[scenario_name].get(backend)
+                if s is None:
+                    continue
+                cfg = s["config"]
+                m = s["metrics"]
+                status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
+                f.write(
+                    f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
+                    f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
+                    f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
+                    f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
+                    f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
+                )
+
+        f.write("\n## Notes\n\n")
+        f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
+        f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
+
+    print(f"Wrote comparison to {args.output}")
+
+
+if __name__ == "__main__":
+    main()
--- a/experiments/dsv4_h200_256k_4k_probe/config.env
+++ b/experiments/dsv4_h200_256k_4k_probe/config.env
@ -0,0 +1,32 @@
+# OOM probe for 256k input + 4k output on DeepSeek-V4-Flash (H200).
+# Tests whether SGLang and vLLM can serve this context length without OOM
+# at concurrency=2 before adding it to the full long-context matrix.
+
+EXPERIMENT="dsv4_h200_256k_4k_probe"
+MODEL_NAME="DeepSeek-V4-Flash"
+MODEL_PATH="/data/models/DeepSeek-V4-Flash"
+SERVED_MODEL_NAME="deepseek-v4-flash"
+
+SGLANG_PORT="${SGLANG_PORT:-30006}"
+VLLM_PORT="${VLLM_PORT:-30005}"
+
+VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
+VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
+
+export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
+TP=8
+
+# 256k input + 4k output + padding -> use 270000.
+MAX_MODEL_LEN=270000
+MAX_NUM_SEQS=8
+MAX_RUNNING=8
+
+# "concurrency input_len output_len num_prompts"
+declare -a SCENARIOS=(
+  "2 262144 4096 4"
+)
+
+VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
+
+SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh"
+VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh"
--- a/experiments/dsv4_h200_256k_4k_probe/parse_backend.py
+++ b/experiments/dsv4_h200_256k_4k_probe/parse_backend.py
@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""Parse raw sglang.bench_serving JSONL outputs for one backend.
+
+Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
+and generates results.json + report.md in the given result root.
+
+Usage:
+    python3 parse_backend.py <result_root> [--backend sglang|vllm]
+"""
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+
+def parse_jsonl(path: Path) -> dict | None:
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                return json.loads(line)
+            except json.JSONDecodeError:
+                continue
+    return None
+
+
+def compute_metrics(data: dict) -> dict:
+    completed = data.get("completed", 0)
+    total = len(data.get("input_lens", []))
+    failed = total - completed if total > 0 else 0
+    duration_s = data.get("duration", 0.0)
+
+    return {
+        "success": completed,
+        "failed": failed,
+        "duration_s": duration_s,
+        "request_throughput": data.get("request_throughput", 0.0),
+        "input_token_throughput": data.get("input_throughput", 0.0),
+        "output_token_throughput": data.get("output_throughput", 0.0),
+        "total_token_throughput": data.get("total_throughput", 0.0),
+        "total_input_tokens": data.get("total_input_tokens", 0),
+        "total_output_tokens": data.get("total_output_tokens", 0),
+        "e2e_ms": {
+            "mean": data.get("mean_e2e_latency_ms", 0.0),
+            "p50": data.get("median_e2e_latency_ms", 0.0),
+            "p90": data.get("p90_e2e_latency_ms", 0.0),
+            "p95": data.get("p95_e2e_latency_ms", 0.0),
+            "p99": data.get("p99_e2e_latency_ms", 0.0),
+        },
+        "ttft_ms": {
+            "mean": data.get("mean_ttft_ms", 0.0),
+            "p50": data.get("median_ttft_ms", 0.0),
+            "p90": data.get("p90_ttft_ms", 0.0),
+            "p95": data.get("p95_ttft_ms", 0.0),
+            "p99": data.get("p99_ttft_ms", 0.0),
+        },
+        "tpot_ms": {
+            "mean": data.get("mean_tpot_ms", 0.0),
+            "p50": data.get("median_tpot_ms", 0.0),
+            "p90": data.get("p90_tpot_ms", 0.0),
+            "p95": data.get("p95_tpot_ms", 0.0),
+            "p99": data.get("p99_tpot_ms", 0.0),
+        },
+        "itl_ms": {
+            "mean": data.get("mean_itl_ms", 0.0),
+            "p50": data.get("median_itl_ms", 0.0),
+            "p90": data.get("p90_itl_ms", 0.0),
+            "p95": data.get("p95_itl_ms", 0.0),
+            "p99": data.get("p99_itl_ms", 0.0),
+        },
+    }
+
+
+def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
+    return f"c{concurrency}_i{input_len}_o{output_len}"
+
+
+def slo_status(metrics: dict) -> dict:
+    """Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
+    ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
+    tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
+    if ttft_ok and tpot_ok:
+        mark = "✅"
+    elif ttft_ok or tpot_ok:
+        mark = "⚠️"
+    else:
+        mark = "❌"
+    return {
+        "ttft_p95_ok": ttft_ok,
+        "tpot_mean_ok": tpot_ok,
+        "overall": mark,
+    }
+
+
+def append_scenario(results_json: Path, scenario: dict) -> None:
+    with open(results_json, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    data["scenarios"].append(scenario)
+    with open(results_json, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2, ensure_ascii=False)
+
+
+def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
+    report_path = result_root / "report.md"
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
+        f.write(f"- Result root: `{result_root}`\n")
+        f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
+        f.write(f"- Backend: {backend.upper()} (TP=8)\n")
+        f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
+
+        f.write("## Results\n\n")
+        f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
+        f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
+
+        for s in scenarios:
+            cfg = s["config"]
+            m = s["metrics"]
+            slo = s.get("slo_status", {}).get("overall", "")
+            f.write(
+                f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
+                f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
+                f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
+                f"{m['total_token_throughput']:.2f} | "
+                f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
+                f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
+                f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
+            )
+        f.write("\n")
+        f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("result_root", type=Path)
+    parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
+    args = parser.parse_args()
+
+    result_root = args.result_root
+    raw_dir = result_root / "raw_outputs"
+    results_json = result_root / "results.json"
+
+    if not raw_dir.exists():
+        raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
+
+    backend = args.backend
+    if backend is None:
+        # Infer from filenames if not provided.
+        for p in raw_dir.iterdir():
+            if p.name.startswith("sglang_"):
+                backend = "sglang"
+                break
+            if p.name.startswith("vllm_"):
+                backend = "vllm"
+                break
+    if backend is None:
+        raise SystemExit("Could not infer backend from raw outputs")
+
+    scenarios = []
+    for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
+        parts = jsonl_path.stem.split("_")
+        if len(parts) < 5:
+            continue
+
+        label = parts[1]
+        if label == "sharegpt":
+            phase = "sharegpt"
+            dataset = "sharegpt"
+        else:
+            phase = label
+            dataset = "random"
+
+        # The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
+        try:
+            concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
+        except ValueError:
+            continue
+
+        data = parse_jsonl(jsonl_path)
+        if data is None:
+            continue
+
+        metrics = compute_metrics(data)
+        scenario = {
+            "name": scenario_name(concurrency, input_len, output_len),
+            "config": {
+                "phase": phase,
+                "concurrency": concurrency,
+                "input_len": input_len,
+                "output_len": output_len,
+                "dataset": dataset,
+                "num_prompts": metrics["success"] + metrics["failed"],
+            },
+            "metrics": metrics,
+            "slo_status": slo_status(metrics),
+            "raw_file": str(jsonl_path),
+        }
+        scenarios.append(scenario)
+
+    if not scenarios:
+        print("No benchmark outputs found to parse")
+        return
+
+    if results_json.exists():
+        with open(results_json, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        data["scenarios"] = scenarios
+        with open(results_json, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+
+    generate_report(result_root, backend, scenarios)
+    print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
+
+
+if __name__ == "__main__":
+    main()
--- a/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/comparison.md
+++ b/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/comparison.md
@ -0,0 +1,21 @@
+# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)
+
+## Summary
+
+- Model: `/data/models/DeepSeek-V4-Flash`
+- Hardware: 8x NVIDIA H200 143GB
+- Tensor Parallelism: 8
+- Benchmark client: `sglang.bench_serving`
+- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)
+
+## Side-by-side results
+
+| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |
+|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| c2_i262144_o4096 | sglang | 2 | 262144 | 4096 | 0.07 | 165.58 | 9734.97 | 10056.38 | 9.32 | 12.82 | 13.41 | 38983.24 | ⚠️ |
+| c2_i262144_o4096 | vllm | 2 | 262144 | 4096 | 0.08 | 180.27 | 8742.36 | 9023.10 | 8.64 | 11.56 | 12.05 | 35853.05 | ⚠️ |
+
+## Notes
+
+- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).
+- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.
--- a/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang/report.md
+++ b/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang/report.md
@ -0,0 +1,15 @@
+# H200 SGLANG Comparison Benchmark Report
+
+- Result root: `/data/user1/yy/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang`
+- Model: `/data/models/DeepSeek-V4-Flash`
+- Backend: SGLANG (TP=8)
+- Benchmark client: `sglang.bench_serving --backend sglang`
+
+## Results
+
+| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| c2_i262144_o4096 | 64k | 2 | 262144 | 4096 | 54.92 | 4 | 0.07 | 9185.73 | 165.58 | 9351.31 | 6239.82 | 9734.97 | 10056.38 | 9.32 | 12.82 | 13.41 | 26116.39 | 38290.97 | 38983.24 | ⚠️ |
+
+SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
+
--- a/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang/results.json
+++ b/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang/results.json
@ -0,0 +1,84 @@
+{
+  "metadata": {
+    "experiment": "dsv4_h200_256k_4k_probe_sglang",
+    "run_id": "20260708-094838",
+    "timestamp": "2026-07-08T09:48:42+00:00",
+    "model": "/data/models/DeepSeek-V4-Flash",
+    "backend": "sglang",
+    "engine": "sglang",
+    "hardware": "8x NVIDIA H200 143GB",
+    "accelerator": "NVIDIA H200",
+    "chip": "nvidia_h200",
+    "script": "experiments/dsv4_h200_256k_4k_probe/run_bench.sh",
+    "env": "/data/user1/yy/envs/sglang",
+    "git_commit": "a9d87f5",
+    "git_dirty": "dirty",
+    "description": "H200 64k-context sglang TP=8 comparison for DeepSeek-V4-Flash"
+  },
+  "config": {
+    "tp": 8,
+    "cuda_visible_devices": "0,1,2,3,4,5,6,7",
+    "max_model_len": 70000,
+    "backend": "sglang",
+    "server_start_script": "experiments/dsv4_h200_256k_4k_probe/start_sglang.sh",
+    "server_args": "sglang serve --trust-remote-code --model-path /data/models/DeepSeek-V4-Flash --tp 8 --moe-runner-backend marlin --context-length 270000 --max-running-requests 8 --mem-fraction-static 0.88 --host 0.0.0.0 --port 30006"
+  },
+  "scenarios": [
+    {
+      "name": "c2_i262144_o4096",
+      "config": {
+        "phase": "64k",
+        "concurrency": 2,
+        "input_len": 262144,
+        "output_len": 4096,
+        "dataset": "random",
+        "num_prompts": 4
+      },
+      "metrics": {
+        "success": 4,
+        "failed": 0,
+        "duration_s": 54.917225173005136,
+        "request_throughput": 0.07283689202065187,
+        "input_token_throughput": 9185.733591069486,
+        "output_token_throughput": 165.57646478594688,
+        "total_token_throughput": 9351.310055855432,
+        "total_input_tokens": 504455,
+        "total_output_tokens": 9093,
+        "e2e_ms": {
+          "mean": 26116.388653255854,
+          "p50": 24796.823772507196,
+          "p90": 37425.63136690442,
+          "p95": 38290.972255454835,
+          "p99": 38983.24496629517
+        },
+        "ttft_ms": {
+          "mean": 6239.822703748359,
+          "p50": 6964.9924714976805,
+          "p90": 9333.200673892861,
+          "p95": 9734.969091441597,
+          "p99": 10056.383825480587
+        },
+        "tpot_ms": {
+          "mean": 9.319386968312292,
+          "p50": 8.102992777237152,
+          "p90": 12.085884372263013,
+          "p95": 12.818917980381144,
+          "p99": 13.40534486687565
+        },
+        "itl_ms": {
+          "mean": 8.747509787324509,
+          "p50": 7.522233994677663,
+          "p90": 7.900186805636622,
+          "p95": 8.008256598259322,
+          "p99": 8.26424767961726
+        }
+      },
+      "slo_status": {
+        "ttft_p95_ok": false,
+        "tpot_mean_ok": true,
+        "overall": "⚠️"
+      },
+      "raw_file": "/data/user1/yy/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/sglang/raw_outputs/sglang_64k_0708_2_262144_4096.jsonl"
+    }
+  ]
+}
--- a/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm/report.md
+++ b/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm/report.md
@ -0,0 +1,15 @@
+# H200 VLLM Comparison Benchmark Report
+
+- Result root: `/data/user1/yy/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm`
+- Model: `/data/models/DeepSeek-V4-Flash`
+- Backend: VLLM (TP=8)
+- Benchmark client: `sglang.bench_serving --backend vllm`
+
+## Results
+
+| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| c2_i262144_o4096 | 64k | 2 | 262144 | 4096 | 50.44 | 4 | 0.08 | 10000.93 | 180.27 | 10181.20 | 5506.23 | 8742.36 | 9023.10 | 8.64 | 11.56 | 12.05 | 24057.58 | 35278.29 | 35853.05 | ⚠️ |
+
+SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
+
--- a/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm/results.json
+++ b/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm/results.json
@ -0,0 +1,84 @@
+{
+  "metadata": {
+    "experiment": "dsv4_h200_256k_4k_probe_vllm",
+    "run_id": "20260708-094838",
+    "timestamp": "2026-07-08T09:48:42+00:00",
+    "model": "/data/models/DeepSeek-V4-Flash",
+    "backend": "vllm",
+    "engine": "vllm",
+    "hardware": "8x NVIDIA H200 143GB",
+    "accelerator": "NVIDIA H200",
+    "chip": "nvidia_h200",
+    "script": "experiments/dsv4_h200_256k_4k_probe/run_bench.sh",
+    "env": "/data/user1/yy/envs/vllm",
+    "git_commit": "a9d87f5",
+    "git_dirty": "dirty",
+    "description": "H200 64k-context vllm TP=8 comparison for DeepSeek-V4-Flash"
+  },
+  "config": {
+    "tp": 8,
+    "cuda_visible_devices": "0,1,2,3,4,5,6,7",
+    "max_model_len": 70000,
+    "backend": "vllm",
+    "server_start_script": "experiments/dsv4_h200_256k_4k_probe/start_vllm.sh",
+    "server_args": "vllm serve /data/models/DeepSeek-V4-Flash --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len 270000 --max-num-seqs 8 --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port 30005"
+  },
+  "scenarios": [
+    {
+      "name": "c2_i262144_o4096",
+      "config": {
+        "phase": "64k",
+        "concurrency": 2,
+        "input_len": 262144,
+        "output_len": 4096,
+        "dataset": "random",
+        "num_prompts": 4
+      },
+      "metrics": {
+        "success": 4,
+        "failed": 0,
+        "duration_s": 50.4408314510074,
+        "request_throughput": 0.07930083396593401,
+        "input_token_throughput": 10000.92554957131,
+        "output_token_throughput": 180.27062081305948,
+        "total_token_throughput": 10181.19617038437,
+        "total_input_tokens": 504455,
+        "total_output_tokens": 9093,
+        "e2e_ms": {
+          "mean": 24057.57891399844,
+          "p50": 22897.005692502717,
+          "p90": 34559.83010400087,
+          "p95": 35278.28712599876,
+          "p99": 35853.052743597065
+        },
+        "ttft_ms": {
+          "mean": 5506.22531299814,
+          "p50": 6125.462318494101,
+          "p90": 8391.44238250883,
+          "p95": 8742.361543760488,
+          "p99": 9023.096872761815
+        },
+        "tpot_ms": {
+          "mean": 8.641820628886816,
+          "p50": 7.61968608874154,
+          "p90": 10.948930343003886,
+          "p95": 11.562458186103475,
+          "p99": 12.053280460583151
+        },
+        "itl_ms": {
+          "mean": 8.202191201391031,
+          "p50": 7.1586980047868565,
+          "p90": 7.325935200788081,
+          "p95": 7.377560806344263,
+          "p99": 7.590774544223677
+        }
+      },
+      "slo_status": {
+        "ttft_p95_ok": false,
+        "tpot_mean_ok": true,
+        "overall": "⚠️"
+      },
+      "raw_file": "/data/user1/yy/experiments/dsv4_h200_256k_4k_probe/results/20260708-094838/vllm/raw_outputs/vllm_64k_0708_2_262144_4096.jsonl"
+    }
+  ]
+}
--- a/experiments/dsv4_h200_256k_4k_probe/run_bench.sh
+++ b/experiments/dsv4_h200_256k_4k_probe/run_bench.sh
@ -0,0 +1,275 @@
+#!/usr/bin/env bash
+# 64k-context SGLang vs vLLM comparison on H200.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
+
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
+RESULT_BASE="${SCRIPT_DIR}/results"
+
+log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
+mkdir -p "$log_dir_global"
+log_init "${log_dir_global}/orchestrator.log"
+
+log "experiment=${EXPERIMENT_NAME}"
+log "run_id=${RUN_ID}"
+log "platform=${PLATFORM}"
+log "hardware=${HARDWARE}"
+log "model=${MODEL_PATH}"
+log "max_model_len=${MAX_MODEL_LEN}"
+log "scenarios=${#SCENARIOS[@]}"
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+is_server_healthy() {
+  local port="$1"
+  curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
+}
+
+stop_server() {
+  local backend="$1"
+  local pid_file="/data/user1/yy/dsv4_h200_256k_4k_probe_${backend}.pid"
+  if [[ -f "$pid_file" ]]; then
+    local pid
+    pid="$(cat "$pid_file")"
+    if kill -0 "$pid" 2>/dev/null; then
+      log "stopping ${backend} server pid=${pid}"
+      kill "$pid" 2>/dev/null || true
+      sleep 5
+      kill -9 "$pid" 2>/dev/null || true
+    fi
+    rm -f "$pid_file"
+  fi
+  # Fallback cleanup.
+  if [[ "$backend" == "sglang" ]]; then
+    pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true
+  else
+    pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
+  fi
+  sleep 2
+}
+
+start_server() {
+  local backend="$1"
+  local start_script
+  if [[ "$backend" == "sglang" ]]; then
+    start_script="${SGLANG_START_SCRIPT}"
+    local port="$SGLANG_PORT"
+  else
+    start_script="${VLLM_START_SCRIPT}"
+    local port="$VLLM_PORT"
+  fi
+
+  log "starting ${backend} server with ${start_script}"
+  bash "${start_script}" >> "${log_dir_global}/${backend}.server.outer.log" 2>&1
+
+  if ! is_server_healthy "$port"; then
+    log "error: ${backend} server failed to become healthy"
+    return 1
+  fi
+  log "${backend} server is healthy on port ${port}"
+}
+
+run_warmup() {
+  local backend="$1"
+  local port input_len output_len num
+  if [[ "$backend" == "sglang" ]]; then
+    port="$SGLANG_PORT"
+  else
+    port="$VLLM_PORT"
+  fi
+
+  # Warmup with a moderate 64k request to exercise the long-context path.
+  input_len=65536
+  output_len=256
+  num=1
+
+  log "warming up ${backend} (input=${input_len}, output=${output_len}, num=${num})"
+  "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \
+    --backend "$backend" \
+    --port "$port" \
+    --input-len "$input_len" \
+    --output-len "$output_len" \
+    --num "$num" \
+    --env-python "${VENV_CLIENT}/bin/python" \
+    >> "${log_dir_global}/${backend}.warmup.log" 2>&1
+  log "warmup for ${backend} completed"
+}
+
+scenario_already_completed() {
+  local output_file="$1"
+  local expected="$2"
+  [[ -s "$output_file" ]] || return 1
+  local completed
+  completed="$(${VENV_CLIENT}/bin/python -c "
+import json, sys
+path = sys.argv[1]
+try:
+    with open(path, 'r', encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                data = json.loads(line)
+                print(data.get('completed', 0))
+                break
+except Exception:
+    print(0)
+" "$output_file")"
+  [[ "${completed:-0}" -ge "$expected" ]]
+}
+
+run_benchmark() {
+  local backend="$1"
+  local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
+  local raw_dir="${result_root}/raw_outputs"
+  local phase_log_dir="${result_root}/logs"
+
+  mkdir -p "$raw_dir" "$phase_log_dir"
+
+  local port
+  if [[ "$backend" == "sglang" ]]; then
+    port="$SGLANG_PORT"
+  else
+    port="$VLLM_PORT"
+  fi
+
+  log "===== ${backend} START (max_model_len=${MAX_MODEL_LEN}) ====="
+
+  stop_server "$backend"
+  start_server "$backend"
+  run_warmup "$backend"
+
+  for scenario in "${SCENARIOS[@]}"; do
+    read -r concurrency input_len output_len num_prompts <<< "$scenario"
+    output_file="${raw_dir}/${backend}_64k_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
+    detail_log="${phase_log_dir}/${backend}_64k_c${concurrency}_i${input_len}_o${output_len}.log"
+
+    if scenario_already_completed "$output_file" "$num_prompts"; then
+      log "skipping already-completed ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len}"
+      continue
+    fi
+
+    log "running ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
+
+    "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
+      --backend "$backend" \
+      --host 127.0.0.1 \
+      --port "$port" \
+      --dataset-name random \
+      --random-input-len "$input_len" \
+      --random-output-len "$output_len" \
+      --num-prompts "$num_prompts" \
+      --max-concurrency "$concurrency" \
+      --request-rate 10000 \
+      --output-file "$output_file" \
+      --output-details \
+      > "$detail_log" 2>&1 || {
+        log "ERROR: ${backend} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
+        continue
+      }
+
+    log "finished ${backend} scenario: output=${output_file}"
+  done
+
+  stop_server "$backend"
+  log "===== ${backend} DONE ====="
+}
+
+parse_backend() {
+  local backend="$1"
+  local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
+  log "parsing ${backend} results in ${result_root}"
+  "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \
+    >> "${result_root}/logs/parse.log" 2>&1 || {
+      log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log"
+    }
+}
+
+write_backend_metadata() {
+  local backend="$1"
+  local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
+  ensure_result_root "$result_root"
+  local meta_json="${result_root}/results.json"
+
+  local env_path
+  if [[ "$backend" == "sglang" ]]; then
+    env_path="$VENV_SGLANG"
+  else
+    env_path="$VENV_VLLM"
+  fi
+
+  # Capture the exact server launch args for reproducibility.
+  local server_args
+  if [[ "$backend" == "sglang" ]]; then
+    server_args="sglang serve --trust-remote-code --model-path $MODEL_PATH --tp $TP --moe-runner-backend marlin --context-length $MAX_MODEL_LEN --max-running-requests $MAX_RUNNING --mem-fraction-static 0.88 --host 0.0.0.0 --port $SGLANG_PORT"
+  else
+    server_args="vllm serve $MODEL_PATH --trust-remote-code --tensor-parallel-size $TP --kv-cache-dtype fp8 --max-model-len $MAX_MODEL_LEN --max-num-seqs $MAX_NUM_SEQS --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $VLLM_PORT"
+  fi
+
+  write_metadata_json \
+    "$meta_json" \
+    "${EXPERIMENT_NAME}_${backend}" \
+    "$RUN_ID" \
+    "$MODEL_PATH" \
+    "$backend" \
+    "$backend" \
+    "$HARDWARE" \
+    "$ACCELERATOR" \
+    "$CHIP" \
+    "experiments/${EXPERIMENT_NAME}/run_bench.sh" \
+    "$env_path" \
+    "H200 64k-context ${backend} TP=8 comparison for DeepSeek-V4-Flash"
+
+  # Embed config and server args.
+  jq --arg backend "$backend" \
+     --arg server_args "$server_args" \
+     '.config = {
+        "tp": 8,
+        "cuda_visible_devices": "0,1,2,3,4,5,6,7",
+        "max_model_len": 70000,
+        "backend": $backend,
+        "server_start_script": "experiments/dsv4_h200_256k_4k_probe/start_\($backend).sh",
+        "server_args": $server_args
+      }' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json"
+}
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+# Cleanup any leftovers.
+stop_server sglang
+stop_server vllm
+
+write_backend_metadata sglang
+write_backend_metadata vllm
+
+# Run SGLang.
+run_benchmark sglang
+parse_backend sglang
+
+# Run vLLM.
+run_benchmark vllm
+parse_backend vllm
+
+# Generate comparison.
+log "generating comparison report"
+"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \
+  --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \
+  --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \
+  --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
+  >> "${log_dir_global}/compare.log" 2>&1 || {
+    log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
+  }
+
+log "all results saved to ${RESULT_BASE}/${RUN_ID}"
--- a/experiments/dsv4_h200_256k_4k_probe/start_sglang.sh
+++ b/experiments/dsv4_h200_256k_4k_probe/start_sglang.sh
@ -0,0 +1,62 @@
+#!/bin/bash
+# Start SGLang server for the 64k-context comparison.
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+cd /data/user1/yy
+mkdir -p logs
+
+export PATH="${VENV_SGLANG}/bin:$PATH"
+export PYTHONUNBUFFERED=1
+export SGLANG_LOG_LEVEL=info
+export TMPDIR=/data/user1/yy/tmp
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
+
+LOG="/data/user1/yy/logs/dsv4_h200_256k_4k_probe_sglang_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="/data/user1/yy/dsv4_h200_256k_4k_probe_sglang.pid"
+
+rm -f "$PID_FILE"
+
+echo "=== Starting SGLang server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_running=$MAX_RUNNING) ==="
+echo "Model: $MODEL_PATH"
+echo "Port: $SGLANG_PORT"
+echo "Log: $LOG"
+
+nohup sglang serve \
+    --trust-remote-code \
+    --model-path "$MODEL_PATH" \
+    --tp "$TP" \
+    --moe-runner-backend marlin \
+    --context-length "$MAX_MODEL_LEN" \
+    --max-running-requests "$MAX_RUNNING" \
+    --mem-fraction-static 0.88 \
+    --host 0.0.0.0 \
+    --port "$SGLANG_PORT" \
+    > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health..."
+
+for i in $(seq 1 240); do
+    if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then
+        echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}"
+        echo "Log: $LOG"
+        exit 0
+    fi
+    if ! kill -0 $PID 2>/dev/null; then
+        echo "ERROR: SGLang server exited early"
+        tail -200 "$LOG"
+        exit 1
+    fi
+    echo "Waiting... ($i/240)"
+    sleep 5
+done
+
+echo "ERROR: SGLang server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1
--- a/experiments/dsv4_h200_256k_4k_probe/start_vllm.sh
+++ b/experiments/dsv4_h200_256k_4k_probe/start_vllm.sh
@ -0,0 +1,65 @@
+#!/bin/bash
+# Start vLLM server for the 64k-context comparison.
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${SCRIPT_DIR}/config.env"
+
+cd /data/user1/yy
+mkdir -p logs
+
+VENV="${VENV_VLLM}"
+export PATH="$VENV/bin:$PATH"
+export PYTHONUNBUFFERED=1
+export TMPDIR=/data/user1/yy/tmp
+export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
+
+LOG="/data/user1/yy/logs/dsv4_h200_256k_4k_probe_vllm_$(date +%Y%m%d_%H%M%S).log"
+PID_FILE="/data/user1/yy/dsv4_h200_256k_4k_probe_vllm.pid"
+
+rm -f "$PID_FILE"
+
+echo "=== Starting vLLM server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_num_seqs=$MAX_NUM_SEQS) ==="
+echo "Model: $MODEL_PATH"
+echo "Port: $VLLM_PORT"
+echo "Log: $LOG"
+
+nohup vllm serve "$MODEL_PATH" \
+    --trust-remote-code \
+    --tensor-parallel-size "$TP" \
+    --kv-cache-dtype fp8 \
+    --max-model-len "$MAX_MODEL_LEN" \
+    --max-num-seqs "$MAX_NUM_SEQS" \
+    --block-size 256 \
+    --gpu-memory-utilization 0.90 \
+    --tokenizer-mode deepseek_v4 \
+    --reasoning-parser deepseek_v4 \
+    --no-disable-hybrid-kv-cache-manager \
+    --disable-uvicorn-access-log \
+    --port "$VLLM_PORT" \
+    > "$LOG" 2>&1 &
+
+PID=$!
+echo $PID > "$PID_FILE"
+echo "PID: $PID"
+echo "Waiting for health..."
+
+for i in $(seq 1 240); do
+    if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
+        echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
+        echo "Log: $LOG"
+        exit 0
+    fi
+    if ! kill -0 $PID 2>/dev/null; then
+        echo "ERROR: vLLM server exited early"
+        tail -200 "$LOG"
+        exit 1
+    fi
+    echo "Waiting... ($i/240)"
+    sleep 5
+done
+
+echo "ERROR: vLLM server not healthy after 240 retries"
+tail -200 "$LOG"
+exit 1
--- a/experiments/dsv4_h200_256k_4k_probe/warmup.py
+++ b/experiments/dsv4_h200_256k_4k_probe/warmup.py
@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""Send a small number of warmup requests to a running backend.
+
+Uses sglang.bench_serving with a single request so that the same code path
+(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
+benchmark begins. Discards the output.
+"""
+import argparse
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
+        cmd = [
+            str(env_python),
+            "-m",
+            "sglang.bench_serving",
+            "--backend",
+            backend,
+            "--host",
+            host,
+            "--port",
+            str(port),
+            "--dataset-name",
+            "random",
+            "--random-input-len",
+            str(input_len),
+            "--random-output-len",
+            str(output_len),
+            "--num-prompts",
+            str(num),
+            "--max-concurrency",
+            "1",
+            "--request-rate",
+            "10000",
+            "--output-file",
+            tmp.name,
+            "--output-details",
+        ]
+        print(f"[warmup] {' '.join(cmd)}", flush=True)
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if result.returncode != 0:
+            print("[warmup] FAILED", file=sys.stderr)
+            print(result.stdout, file=sys.stderr)
+            print(result.stderr, file=sys.stderr)
+            sys.exit(1)
+        print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Warmup a serving backend.")
+    parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
+    parser.add_argument("--port", type=int, required=True)
+    parser.add_argument("--input-len", type=int, required=True)
+    parser.add_argument("--output-len", type=int, required=True)
+    parser.add_argument("--num", type=int, default=1)
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
+    args = parser.parse_args()
+
+    run_warmup(
+        backend=args.backend,
+        host=args.host,
+        port=args.port,
+        input_len=args.input_len,
+        output_len=args.output_len,
+        num=args.num,
+        env_python=Path(args.env_python),
+    )
+
+
+if __name__ == "__main__":
+    main()