diff --git a/experiments/dsv4_h200_long_context_matrix/compare.py b/experiments/dsv4_h200_long_context_matrix/compare.py new file mode 100755 index 0000000..9445c3d --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/compare.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of SGLang, vLLM, and DSpark results. + +Usage: + python3 compare.py --sglang --vllm --dspark \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sglang", type=Path, required=True) + parser.add_argument("--vllm", type=Path, required=True) + parser.add_argument("--dspark", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + sglang_data = load_result(args.sglang) + vllm_data = load_result(args.vllm) + dspark_data = load_result(args.dspark) + + model = sglang_data.get("metadata", {}).get("model", "unknown") + hardware = sglang_data.get("metadata", {}).get("hardware", "unknown") + + # Map engine names to display labels. + engine_to_label = { + "sglang": "sglang", + "vllm": "vllm", + "vllm-dspark": "dspark", + } + + by_scenario = defaultdict(dict) + for data in (sglang_data, vllm_data, dspark_data): + engine = data["metadata"]["engine"] + label = engine_to_label.get(engine, engine) + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang vs vLLM vs DSpark long-context matrix ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write("- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + f.write("## Side-by-side results\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("sglang", "vllm", "dspark"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/dsv4_h200_long_context_matrix/config.env b/experiments/dsv4_h200_long_context_matrix/config.env index b82fd47..fcddcf5 100644 --- a/experiments/dsv4_h200_long_context_matrix/config.env +++ b/experiments/dsv4_h200_long_context_matrix/config.env @@ -10,9 +10,15 @@ SERVED_MODEL_NAME="deepseek-v4-flash" SGLANG_PORT="${SGLANG_PORT:-30006}" VLLM_PORT="${VLLM_PORT:-30005}" +DSPARK_PORT="${DSPARK_PORT:-30013}" VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" +VENV_VLLM_DSPARK="${VENV_VLLM_DSPARK:-/data/user1/yy/envs/vllm-dspark}" + +# DSpark uses a separate model checkpoint and speculative decoding. +DSPARK_MODEL_NAME="DeepSeek-V4-Flash-DSpark" +DSPARK_MODEL_PATH="/data/models/DeepSeek-V4-Flash-DSpark" export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" TP=8 @@ -60,3 +66,4 @@ VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" +DSPARK_START_SCRIPT="${SCRIPT_DIR:-.}/start_dspark.sh" diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md new file mode 100644 index 0000000..5a4b0a1 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/comparison.md @@ -0,0 +1,83 @@ +# SGLang vs vLLM vs DSpark long-context matrix (8x NVIDIA H200 143GB) + +## Summary + +- Model: `/data/models/DeepSeek-V4-Flash` +- DSpark model: `/data/models/DeepSeek-V4-Flash-DSpark` +- Hardware: 8x NVIDIA H200 143GB +- Benchmark client: `sglang.bench_serving` +- DSpark flags: `--spec-method dspark --spec-model --spec-tokens 5` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +## Side-by-side results + +| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO | +|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c10_i131072_o1024 | sglang | 10 | 131072 | 1024 | 0.68 | 319.74 | 4147.51 | 7409.82 | 25.69 | 49.89 | 58.92 | 26432.66 | ⚠️ | +| c10_i131072_o1024 | vllm | 10 | 131072 | 1024 | 0.39 | 183.08 | 6596.15 | 6603.12 | 47.10 | 77.53 | 84.06 | 48426.64 | ⚠️ | +| c10_i131072_o1024 | dspark | 10 | 131072 | 1024 | 0.66 | 310.66 | 11615.81 | 11676.24 | 24.58 | 46.24 | 46.56 | 29271.39 | ⚠️ | +| c10_i131072_o256 | sglang | 10 | 131072 | 256 | 0.34 | 42.70 | 26046.62 | 26064.38 | 1374.50 | 1810.56 | 19553.62 | 58326.41 | ❌ | +| c10_i131072_o256 | vllm | 10 | 131072 | 256 | 0.38 | 48.09 | 23250.03 | 24238.33 | 150.22 | 299.91 | 301.72 | 51439.08 | ❌ | +| c10_i131072_o256 | dspark | 10 | 131072 | 256 | 0.40 | 51.33 | 24693.20 | 27039.11 | 59.88 | 78.22 | 316.95 | 36603.69 | ❌ | +| c10_i131072_o4096 | sglang | 10 | 131072 | 4096 | 0.27 | 517.96 | 6177.42 | 6920.28 | 14.21 | 19.92 | 25.87 | 54286.63 | ⚠️ | +| c10_i131072_o4096 | vllm | 10 | 131072 | 4096 | 0.23 | 445.81 | 2096.29 | 2168.81 | 18.57 | 21.47 | 21.61 | 67041.13 | ✅ | +| c10_i131072_o4096 | dspark | 10 | 131072 | 4096 | 1.05 | 1998.23 | 1543.09 | 1546.01 | 3.34 | 4.81 | 4.84 | 14731.59 | ✅ | +| c10_i262144_o1024 | sglang | 10 | 262144 | 1024 | 0.16 | 76.30 | 39446.96 | 43965.67 | 108.42 | 202.43 | 202.90 | 120180.87 | ❌ | +| c10_i262144_o1024 | vllm | 10 | 262144 | 1024 | 0.18 | 84.63 | 34130.22 | 38144.40 | 98.87 | 142.57 | 266.08 | 108658.89 | ❌ | +| c10_i262144_o1024 | dspark | 10 | 262144 | 1024 | 0.20 | 96.79 | 39321.30 | 40783.01 | 48.19 | 67.60 | 68.13 | 91123.07 | ⚠️ | +| c10_i262144_o256 | sglang | 10 | 262144 | 256 | 0.15 | 18.70 | 58074.31 | 62652.59 | 2953.70 | 3625.59 | 41185.32 | 134628.33 | ❌ | +| c10_i262144_o256 | vllm | 10 | 262144 | 256 | 0.18 | 22.76 | 50632.44 | 51883.02 | 229.20 | 339.27 | 344.95 | 110216.92 | ❌ | +| c10_i262144_o256 | dspark | 10 | 262144 | 256 | 0.18 | 23.12 | 64654.53 | 67877.62 | 52.05 | 73.84 | 151.28 | 80196.26 | ❌ | +| c10_i262144_o4096 | sglang | 10 | 262144 | 4096 | 0.14 | 272.46 | 18088.12 | 22579.09 | 28.00 | 43.87 | 45.07 | 120183.75 | ⚠️ | +| c10_i262144_o4096 | vllm | 10 | 262144 | 4096 | 0.11 | 215.86 | 34100.32 | 38022.94 | 37.18 | 56.07 | 62.00 | 157944.21 | ⚠️ | +| c10_i262144_o4096 | dspark | 10 | 262144 | 4096 | 0.18 | 349.94 | 33426.83 | 37512.71 | 20.67 | 35.94 | 44.44 | 103440.52 | ⚠️ | +| c25_i65536_o1024 | sglang | 25 | 65536 | 1024 | 0.67 | 346.70 | 19996.43 | 24478.11 | 149.27 | 395.00 | 2037.09 | 69586.94 | ❌ | +| c25_i65536_o1024 | vllm | 25 | 65536 | 1024 | 0.64 | 332.61 | 24861.20 | 28942.75 | 72.36 | 196.24 | 271.32 | 72689.84 | ❌ | +| c25_i65536_o1024 | dspark | 25 | 65536 | 1024 | 0.80 | 412.74 | 26078.12 | 29135.60 | 42.82 | 56.19 | 133.89 | 60995.55 | ⚠️ | +| c25_i65536_o256 | sglang | 25 | 65536 | 256 | 0.71 | 84.45 | 27219.06 | 31612.70 | 292.84 | 801.18 | 938.43 | 68854.26 | ❌ | +| c25_i65536_o256 | vllm | 25 | 65536 | 256 | 0.81 | 95.60 | 24902.72 | 28974.45 | 180.78 | 273.59 | 278.16 | 60781.45 | ❌ | +| c25_i65536_o256 | dspark | 25 | 65536 | 256 | 0.84 | 98.79 | 27635.11 | 29212.48 | 51.68 | 65.52 | 134.94 | 39437.04 | ❌ | +| c25_i65536_o4096 | sglang | 25 | 65536 | 4096 | 0.41 | 950.01 | 18559.58 | 23249.71 | 23.78 | 42.50 | 82.84 | 96670.12 | ⚠️ | +| c25_i65536_o4096 | vllm | 25 | 65536 | 4096 | 0.32 | 741.71 | 24847.97 | 28840.96 | 28.80 | 46.66 | 63.40 | 129654.04 | ⚠️ | +| c25_i65536_o4096 | dspark | 25 | 65536 | 4096 | 0.61 | 1409.20 | 25048.60 | 29143.30 | 14.72 | 30.57 | 42.07 | 73496.43 | ⚠️ | +| c2_i131072_o1024 | sglang | 2 | 131072 | 1024 | 0.60 | 136.10 | 479.99 | 482.72 | 8.56 | 9.72 | 9.79 | 5480.98 | ✅ | +| c2_i131072_o1024 | vllm | 2 | 131072 | 1024 | 0.70 | 158.36 | 676.14 | 695.90 | 6.49 | 8.20 | 8.34 | 4803.24 | ✅ | +| c2_i131072_o1024 | dspark | 2 | 131072 | 1024 | 2.29 | 514.96 | 679.81 | 691.56 | 1.10 | 1.60 | 1.63 | 1450.74 | ✅ | +| c2_i131072_o256 | sglang | 2 | 131072 | 256 | 1.43 | 139.48 | 589.14 | 605.33 | 8.29 | 8.96 | 9.05 | 1750.12 | ✅ | +| c2_i131072_o256 | vllm | 2 | 131072 | 256 | 1.51 | 147.12 | 684.63 | 700.28 | 7.08 | 9.27 | 9.45 | 1556.46 | ✅ | +| c2_i131072_o256 | dspark | 2 | 131072 | 256 | 3.10 | 301.11 | 617.17 | 634.08 | 1.00 | 1.41 | 1.42 | 780.85 | ✅ | +| c2_i131072_o4096 | sglang | 2 | 131072 | 4096 | 0.11 | 244.38 | 584.43 | 599.22 | 7.53 | 7.61 | 7.62 | 28360.35 | ✅ | +| c2_i131072_o4096 | vllm | 2 | 131072 | 4096 | 0.12 | 262.22 | 671.34 | 688.15 | 7.09 | 7.17 | 7.18 | 26496.04 | ✅ | +| c2_i131072_o4096 | dspark | 2 | 131072 | 4096 | 0.47 | 1076.96 | 540.43 | 541.94 | 1.76 | 2.03 | 2.08 | 6313.36 | ✅ | +| c2_i262144_o1024 | sglang | 2 | 262144 | 1024 | 0.51 | 114.94 | 1007.07 | 1024.10 | 8.36 | 9.03 | 9.06 | 6240.65 | ✅ | +| c2_i262144_o1024 | vllm | 2 | 262144 | 1024 | 0.19 | 43.14 | 8722.76 | 9009.70 | 20.17 | 52.24 | 57.91 | 13826.13 | ⚠️ | +| c2_i262144_o1024 | dspark | 2 | 262144 | 1024 | 0.25 | 55.65 | 10466.90 | 10984.16 | 14.36 | 43.30 | 48.52 | 15361.85 | ⚠️ | +| c2_i262144_o256 | sglang | 2 | 262144 | 256 | 0.29 | 28.44 | 9538.91 | 9980.78 | 14.10 | 26.98 | 29.16 | 11941.34 | ⚠️ | +| c2_i262144_o256 | vllm | 2 | 262144 | 256 | 0.23 | 22.74 | 8625.18 | 8952.31 | 10.18 | 23.01 | 25.44 | 10378.60 | ⚠️ | +| c2_i262144_o256 | dspark | 2 | 262144 | 256 | 0.26 | 25.25 | 10459.44 | 10978.82 | 17.58 | 45.28 | 48.94 | 14620.43 | ⚠️ | +| c2_i262144_o4096 | sglang | 2 | 262144 | 4096 | 0.08 | 189.78 | 9453.57 | 9896.38 | 7.82 | 8.48 | 8.60 | 38723.14 | ⚠️ | +| c2_i262144_o4096 | vllm | 2 | 262144 | 4096 | 0.08 | 180.76 | 8759.63 | 9062.80 | 8.63 | 11.53 | 12.01 | 35881.75 | ⚠️ | +| c2_i262144_o4096 | dspark | 2 | 262144 | 4096 | 0.17 | 384.42 | 8709.00 | 8987.60 | 3.23 | 6.44 | 6.99 | 15113.78 | ⚠️ | +| c5_i131072_o1024 | sglang | 5 | 131072 | 1024 | 0.87 | 321.54 | 1231.23 | 1298.36 | 9.03 | 10.47 | 10.71 | 7298.25 | ✅ | +| c5_i131072_o1024 | vllm | 5 | 131072 | 1024 | 0.90 | 331.65 | 856.18 | 917.80 | 8.93 | 10.48 | 10.62 | 6989.27 | ✅ | +| c5_i131072_o1024 | dspark | 5 | 131072 | 1024 | 2.75 | 1019.06 | 1080.05 | 1176.83 | 2.53 | 3.98 | 4.18 | 2383.07 | ✅ | +| c5_i131072_o256 | sglang | 5 | 131072 | 256 | 1.05 | 146.97 | 3270.23 | 4685.65 | 19.91 | 40.81 | 46.10 | 7815.89 | ⚠️ | +| c5_i131072_o256 | vllm | 5 | 131072 | 256 | 0.39 | 54.07 | 9384.23 | 9677.91 | 51.54 | 107.16 | 117.25 | 19860.69 | ❌ | +| c5_i131072_o256 | dspark | 5 | 131072 | 256 | 0.44 | 60.98 | 10128.32 | 10501.28 | 37.60 | 60.86 | 61.78 | 17256.59 | ⚠️ | +| c5_i131072_o4096 | sglang | 5 | 131072 | 4096 | 0.20 | 378.76 | 4667.56 | 6275.51 | 11.98 | 22.07 | 23.60 | 31176.81 | ⚠️ | +| c5_i131072_o4096 | vllm | 5 | 131072 | 4096 | 0.23 | 447.85 | 1054.01 | 1104.66 | 8.48 | 9.13 | 9.14 | 28818.20 | ✅ | +| c5_i131072_o4096 | dspark | 5 | 131072 | 4096 | 0.70 | 1335.09 | 1156.91 | 1249.38 | 3.20 | 4.19 | 4.21 | 11315.66 | ✅ | +| c5_i262144_o1024 | sglang | 5 | 262144 | 1024 | 0.15 | 56.42 | 20522.02 | 25944.29 | 54.11 | 118.98 | 124.02 | 53444.22 | ❌ | +| c5_i262144_o1024 | vllm | 5 | 262144 | 1024 | 0.16 | 60.32 | 22564.89 | 26639.92 | 47.49 | 100.69 | 109.86 | 53790.59 | ⚠️ | +| c5_i262144_o1024 | dspark | 5 | 262144 | 1024 | 0.19 | 69.32 | 22995.12 | 26781.97 | 38.56 | 57.19 | 57.43 | 40847.11 | ⚠️ | +| c5_i262144_o256 | sglang | 5 | 262144 | 256 | 0.16 | 21.94 | 25875.40 | 30596.95 | 97.05 | 231.75 | 252.98 | 43756.53 | ❌ | +| c5_i262144_o256 | vllm | 5 | 262144 | 256 | 0.19 | 25.85 | 21986.99 | 26381.86 | 123.42 | 202.78 | 215.56 | 45761.90 | ❌ | +| c5_i262144_o256 | dspark | 5 | 262144 | 256 | 0.19 | 26.70 | 29425.38 | 30576.66 | 41.74 | 60.42 | 62.17 | 37097.84 | ⚠️ | +| c5_i262144_o4096 | sglang | 5 | 262144 | 4096 | 0.11 | 209.50 | 16539.50 | 20323.40 | 25.05 | 56.69 | 60.22 | 60383.57 | ⚠️ | +| c5_i262144_o4096 | vllm | 5 | 262144 | 4096 | 0.11 | 205.12 | 22171.78 | 26533.69 | 25.69 | 63.32 | 68.33 | 64086.27 | ⚠️ | +| c5_i262144_o4096 | dspark | 5 | 262144 | 4096 | 0.16 | 299.79 | 22250.95 | 26667.05 | 17.46 | 45.41 | 49.74 | 51820.15 | ⚠️ | + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out. diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md new file mode 100644 index 0000000..fdd3345 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/report.md @@ -0,0 +1,35 @@ +# 8x NVIDIA H200 143GB VLLM Benchmark Report + +- Result root: `/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark` +- Model: `/data/models/DeepSeek-V4-Flash-DSpark` +- Backend: VLLM +- Benchmark client: `sglang.bench_serving --backend vllm` + +## Results + +| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c10_i131072_o1024 | dspark | 10 | 131072 | 1024 | 30.45 | 20 | 0.66 | 48423.00 | 310.66 | 48733.65 | 4033.86 | 11615.81 | 11676.24 | 24.58 | 46.24 | 46.56 | 14685.68 | 28877.70 | 29271.39 | ⚠️ | +| c10_i131072_o256 | dspark | 10 | 131072 | 256 | 49.62 | 20 | 0.40 | 29712.25 | 51.33 | 29763.58 | 16856.00 | 24693.20 | 27039.11 | 59.88 | 78.22 | 316.95 | 22010.51 | 32508.36 | 36603.69 | ❌ | +| c10_i131072_o4096 | dspark | 10 | 131072 | 4096 | 19.08 | 20 | 1.05 | 77265.07 | 1998.23 | 79263.30 | 727.34 | 1543.09 | 1546.01 | 3.34 | 4.81 | 4.84 | 7316.24 | 12401.94 | 14731.59 | ✅ | +| c2_i131072_o1024 | dspark | 2 | 131072 | 1024 | 1.75 | 4 | 2.29 | 138490.59 | 514.96 | 139005.55 | 447.72 | 679.81 | 691.56 | 1.10 | 1.60 | 1.63 | 794.56 | 1362.81 | 1450.74 | ✅ | +| c2_i131072_o256 | dspark | 2 | 131072 | 256 | 1.29 | 4 | 3.10 | 187564.60 | 301.11 | 187865.71 | 419.01 | 617.17 | 634.08 | 1.00 | 1.41 | 1.42 | 536.04 | 747.52 | 780.85 | ✅ | +| c2_i131072_o4096 | dspark | 2 | 131072 | 4096 | 8.44 | 4 | 0.47 | 28698.90 | 1076.96 | 29775.86 | 325.51 | 540.43 | 541.94 | 1.76 | 2.03 | 2.08 | 4193.26 | 6216.99 | 6313.36 | ✅ | +| c5_i131072_o1024 | dspark | 5 | 131072 | 1024 | 3.63 | 10 | 2.75 | 203409.39 | 1019.06 | 204428.44 | 595.29 | 1080.05 | 1176.83 | 2.53 | 3.98 | 4.18 | 1507.07 | 2347.99 | 2383.07 | ✅ | +| c5_i131072_o256 | dspark | 5 | 131072 | 256 | 22.89 | 10 | 0.44 | 32258.45 | 60.98 | 32319.42 | 5806.57 | 10128.32 | 10501.28 | 37.60 | 60.86 | 61.78 | 11223.11 | 16310.02 | 17256.59 | ⚠️ | +| c5_i131072_o4096 | dspark | 5 | 131072 | 4096 | 14.28 | 10 | 0.70 | 51732.19 | 1335.09 | 53067.28 | 560.62 | 1156.91 | 1249.38 | 3.20 | 4.19 | 4.21 | 6029.71 | 10552.87 | 11315.66 | ✅ | +| c10_i262144_o1024 | dspark | 10 | 262144 | 1024 | 97.73 | 20 | 0.20 | 28498.71 | 96.79 | 28595.50 | 24591.83 | 39321.30 | 40783.01 | 48.19 | 67.60 | 68.13 | 45387.50 | 70254.49 | 91123.07 | ⚠️ | +| c10_i262144_o256 | dspark | 10 | 262144 | 256 | 110.17 | 20 | 0.18 | 25281.12 | 23.12 | 25304.24 | 42248.63 | 64654.53 | 67877.62 | 52.05 | 73.84 | 151.28 | 48223.85 | 78728.49 | 80196.26 | ❌ | +| c10_i262144_o4096 | dspark | 10 | 262144 | 4096 | 108.97 | 20 | 0.18 | 25559.57 | 349.94 | 25909.50 | 13472.46 | 33426.83 | 37512.71 | 20.67 | 35.94 | 44.44 | 52210.57 | 100105.13 | 103440.52 | ⚠️ | +| c2_i262144_o1024 | dspark | 2 | 262144 | 1024 | 16.19 | 4 | 0.25 | 31154.91 | 55.65 | 31210.55 | 5853.53 | 10466.90 | 10984.16 | 14.36 | 43.30 | 48.52 | 7871.57 | 14740.72 | 15361.85 | ⚠️ | +| c2_i262144_o256 | dspark | 2 | 262144 | 256 | 15.41 | 4 | 0.26 | 32738.24 | 25.25 | 32763.49 | 5850.43 | 10459.44 | 10978.82 | 17.58 | 45.28 | 48.94 | 7672.82 | 14088.43 | 14620.43 | ⚠️ | +| c2_i262144_o4096 | dspark | 2 | 262144 | 4096 | 23.65 | 4 | 0.17 | 21326.49 | 384.42 | 21710.90 | 5487.63 | 8709.00 | 8987.60 | 3.23 | 6.44 | 6.99 | 11596.58 | 14994.53 | 15113.78 | ⚠️ | +| c5_i262144_o1024 | dspark | 5 | 262144 | 1024 | 53.37 | 10 | 0.19 | 26116.34 | 69.32 | 26185.66 | 13406.14 | 22995.12 | 26781.97 | 38.56 | 57.19 | 57.43 | 25295.54 | 38376.69 | 40847.11 | ⚠️ | +| c5_i262144_o256 | dspark | 5 | 262144 | 256 | 52.29 | 10 | 0.19 | 26655.70 | 26.70 | 26682.40 | 17996.01 | 29425.38 | 30576.66 | 41.74 | 60.42 | 62.17 | 23507.98 | 35234.36 | 37097.84 | ⚠️ | +| c5_i262144_o4096 | dspark | 5 | 262144 | 4096 | 63.58 | 10 | 0.16 | 21924.13 | 299.79 | 22223.92 | 10590.48 | 22250.95 | 26667.05 | 17.46 | 45.41 | 49.74 | 30807.30 | 48506.31 | 51820.15 | ⚠️ | +| c25_i65536_o1024 | dspark | 25 | 65536 | 1024 | 62.71 | 50 | 0.80 | 29366.67 | 412.74 | 29779.41 | 11692.07 | 26078.12 | 29135.60 | 42.82 | 56.19 | 133.89 | 29981.59 | 57554.65 | 60995.55 | ⚠️ | +| c25_i65536_o256 | dspark | 25 | 65536 | 256 | 59.86 | 50 | 0.84 | 30764.36 | 98.79 | 30863.14 | 19438.04 | 27635.11 | 29212.48 | 51.68 | 65.52 | 134.94 | 24957.48 | 37092.86 | 39437.04 | ❌ | +| c25_i65536_o4096 | dspark | 25 | 65536 | 4096 | 82.31 | 50 | 0.61 | 22371.88 | 1409.20 | 23781.09 | 8084.05 | 25048.60 | 29143.30 | 14.72 | 30.57 | 42.07 | 38164.69 | 72245.56 | 73496.43 | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json new file mode 100644 index 0000000..d17c50d --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/results.json @@ -0,0 +1,1219 @@ +{ + "metadata": { + "experiment": "dsv4_h200_long_context_matrix_dspark", + "run_id": "20260709-020227", + "timestamp": "2026-07-09T02:02:29+00:00", + "model": "/data/models/DeepSeek-V4-Flash-DSpark", + "backend": "dspark", + "engine": "vllm-dspark", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_long_context_matrix/run_bench.sh", + "env": "/data/user1/yy/envs/vllm-dspark", + "git_commit": "0aeabad", + "git_dirty": "dirty", + "description": "H200 long-context matrix dspark TP=8 for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "dspark", + "server_start_script": "experiments/dsv4_h200_long_context_matrix/start_dspark.sh", + "groups": [ + { + "label": "64k", + "max_model_len": 80000, + "max_slots": 25 + }, + { + "label": "128k", + "max_model_len": 140000, + "max_slots": 10 + }, + { + "label": "256k", + "max_model_len": 270000, + "max_slots": 10 + } + ] + }, + "scenarios": [ + { + "name": "c10_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 30.44836392698926, + "request_throughput": 0.6568497423361428, + "input_token_throughput": 48422.99584750756, + "output_token_throughput": 310.6570856378787, + "total_token_throughput": 48733.652933145444, + "total_input_tokens": 1474401, + "total_output_tokens": 9459, + "e2e_ms": { + "mean": 14685.683184953814, + "p50": 14254.763487013406, + "p90": 27765.363381317, + "p95": 28877.701320237247, + "p99": 29271.392024847442 + }, + "ttft_ms": { + "mean": 4033.855032298016, + "p50": 1451.7911804869073, + "p90": 10187.548868515298, + "p95": 11615.812585884123, + "p99": 11676.239348381641 + }, + "tpot_ms": { + "mean": 24.581426407367566, + "p50": 26.691134347746317, + "p90": 41.43077303036502, + "p95": 46.23705537957289, + "p99": 46.55721496167711 + }, + "itl_ms": { + "mean": 135.43289032106352, + "p50": 21.5837610012386, + "p90": 320.3089441929478, + "p95": 337.2771248104982, + "p99": 389.6573220437858 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_1024.jsonl" + }, + { + "name": "c10_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 49.62265793100232, + "request_throughput": 0.4030416917168955, + "input_token_throughput": 29712.253665454125, + "output_token_throughput": 51.32735944014665, + "total_token_throughput": 29763.581024894273, + "total_input_tokens": 1474401, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 22010.51428525243, + "p50": 23559.51010050194, + "p90": 31866.07759631006, + "p95": 32508.359657706755, + "p99": 36603.693813944985 + }, + "ttft_ms": { + "mean": 16855.99723370542, + "p50": 21033.618335000938, + "p90": 23834.377329907147, + "p95": 24693.19532076188, + "p99": 27039.11317213962 + }, + "tpot_ms": { + "mean": 59.877913532559745, + "p50": 49.298212603349945, + "p90": 60.082798051918886, + "p95": 78.21795468785261, + "p99": 316.9470677312616 + }, + "itl_ms": { + "mean": 219.8073648530414, + "p50": 271.98732501710765, + "p90": 333.0672606069129, + "p95": 357.2517809981946, + "p99": 390.4315355594735 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_256.jsonl" + }, + { + "name": "c10_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 19.08237444801489, + "request_throughput": 1.048087598033722, + "input_token_throughput": 77265.0701314259, + "output_token_throughput": 1998.231410031193, + "total_token_throughput": 79263.3015414571, + "total_input_tokens": 1474401, + "total_output_tokens": 38131, + "e2e_ms": { + "mean": 7316.236398999172, + "p50": 8444.335514504928, + "p90": 11945.644905421068, + "p95": 12401.944660821757, + "p99": 14731.589484146385 + }, + "ttft_ms": { + "mean": 727.3375002972898, + "p50": 600.3662900038762, + "p90": 1411.2119916011582, + "p95": 1543.0946984008187, + "p99": 1546.0051852866309 + }, + "tpot_ms": { + "mean": 3.335079117622894, + "p50": 3.428271593855011, + "p90": 4.07458192727982, + "p95": 4.806456682330838, + "p99": 4.841832708764732 + }, + "itl_ms": { + "mean": 20.002630943069565, + "p50": 15.639238990843296, + "p90": 22.070227304357104, + "p95": 34.30636346893152, + "p99": 165.97390236041974 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_10_131072_4096.jsonl" + }, + { + "name": "c2_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 1.7496567660127766, + "request_throughput": 2.2861626792753422, + "input_token_throughput": 138490.59124447187, + "output_token_throughput": 514.9581435067709, + "total_token_throughput": 139005.54938797862, + "total_input_tokens": 242311, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 794.5647465166985, + "p50": 718.1648885161849, + "p90": 1252.8917442105014, + "p95": 1362.8063466094316, + "p99": 1450.738028528576 + }, + "ttft_ms": { + "mean": 447.72256826399826, + "p50": 459.6335765090771, + "p90": 665.1223058113828, + "p95": 679.8121634114067, + "p99": 691.5640494914259 + }, + "tpot_ms": { + "mean": 1.0965360940272797, + "p50": 1.3248563036316097, + "p90": 1.5558365349002174, + "p95": 1.5993635767368795, + "p99": 1.6341852102062093 + }, + "itl_ms": { + "mean": 11.097457431955263, + "p50": 8.672151016071439, + "p90": 10.805651010014122, + "p95": 16.695154790068035, + "p99": 34.131238552508904 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_1024.jsonl" + }, + { + "name": "c2_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 1.2918802190106362, + "request_throughput": 3.096262285882301, + "input_token_throughput": 187564.60268860657, + "output_token_throughput": 301.1115073020538, + "total_token_throughput": 187865.7141959086, + "total_input_tokens": 242311, + "total_output_tokens": 389, + "e2e_ms": { + "mean": 536.0358724938123, + "p50": 505.109255987918, + "p90": 705.8564373000991, + "p95": 747.5212021527112, + "p99": 780.8530140348012 + }, + "ttft_ms": { + "mean": 419.00899424945237, + "p50": 393.0140794982435, + "p90": 596.0212633013725, + "p95": 617.1667256538057, + "p99": 634.0830955357524 + }, + "tpot_ms": { + "mean": 0.9978323419928723, + "p50": 1.2476157901743834, + "p90": 1.389477322883724, + "p95": 1.4064074303054686, + "p99": 1.4199515162428642 + }, + "itl_ms": { + "mean": 9.17479329441181, + "p50": 9.940474992617965, + "p90": 10.147733002668247, + "p95": 16.18033648992423, + "p99": 18.293717512278818 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_256.jsonl" + }, + { + "name": "c2_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 8.443216187995858, + "request_throughput": 0.4737531185908753, + "input_token_throughput": 28698.897979718397, + "output_token_throughput": 1076.9592768367072, + "total_token_throughput": 29775.857256555104, + "total_input_tokens": 242311, + "total_output_tokens": 9093, + "e2e_ms": { + "mean": 4193.263632994785, + "p50": 4219.779900988215, + "p90": 6096.524830308044, + "p95": 6216.986983161768, + "p99": 6313.3567054447485 + }, + "ttft_ms": { + "mean": 325.5081640018034, + "p50": 354.36446650419384, + "p90": 538.5507072962355, + "p95": 540.4322966438485, + "p99": 541.937568121939 + }, + "tpot_ms": { + "mean": 1.7589143165131924, + "p50": 1.6630257624623965, + "p90": 1.963713117340703, + "p95": 2.027377708667831, + "p99": 2.078309381729534 + }, + "itl_ms": { + "mean": 10.065626301899645, + "p50": 9.872002003248781, + "p90": 10.049426596378908, + "p95": 10.112287598894909, + "p99": 16.348993644351122 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_2_131072_4096.jsonl" + }, + { + "name": "c5_i131072_o1024", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 1024, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 3.6308107929944526, + "request_throughput": 2.754205760127936, + "input_token_throughput": 203409.3876290646, + "output_token_throughput": 1019.0561312473362, + "total_token_throughput": 204428.44376031193, + "total_input_tokens": 738541, + "total_output_tokens": 3700, + "e2e_ms": { + "mean": 1507.069445902016, + "p50": 1389.4180455099558, + "p90": 2304.1342217009515, + "p95": 2347.9891008522827, + "p99": 2383.0730041733477 + }, + "ttft_ms": { + "mean": 595.2868286985904, + "p50": 618.8132940005744, + "p90": 959.0717278100782, + "p95": 1080.0512569068812, + "p99": 1176.834880184324 + }, + "tpot_ms": { + "mean": 2.5274796112419384, + "p50": 2.131028194464456, + "p90": 3.7235818184212732, + "p95": 3.978484070795007, + "p99": 4.182405872693995 + }, + "itl_ms": { + "mean": 16.108580809175006, + "p50": 10.739890494733118, + "p90": 20.27784650272224, + "p95": 23.551662503450643, + "p99": 144.88391225168016 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_1024.jsonl" + }, + { + "name": "c5_i131072_o256", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 22.894497758999933, + "request_throughput": 0.43678617042686396, + "input_token_throughput": 32258.449509322654, + "output_token_throughput": 60.975349391590214, + "total_token_throughput": 32319.424858714247, + "total_input_tokens": 738541, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 11223.107872297987, + "p50": 11322.323512998992, + "p90": 15126.79551889014, + "p95": 16310.015888452468, + "p99": 17256.592184102337 + }, + "ttft_ms": { + "mean": 5806.565075399703, + "p50": 7417.0587029948365, + "p90": 9662.113985186443, + "p95": 10128.318913598194, + "p99": 10501.282856327598 + }, + "tpot_ms": { + "mean": 37.59599971357063, + "p50": 44.28112107098226, + "p90": 59.72495268804145, + "p95": 60.864775370246505, + "p99": 61.77663351601055 + }, + "itl_ms": { + "mean": 216.66011608368717, + "p50": 273.5048829927109, + "p90": 346.02558271435555, + "p95": 364.7101309834396, + "p99": 392.47983420587843 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_256.jsonl" + }, + { + "name": "c5_i131072_o4096", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 131072, + "output_len": 4096, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 14.276236372999847, + "request_throughput": 0.7004647260473114, + "input_token_throughput": 51732.19192397074, + "output_token_throughput": 1335.0857678461755, + "total_token_throughput": 53067.27769181692, + "total_input_tokens": 738541, + "total_output_tokens": 19060, + "e2e_ms": { + "mean": 6029.712195502361, + "p50": 7018.733311502729, + "p90": 9599.38420431863, + "p95": 10552.86903215601, + "p99": 11315.65689442592 + }, + "ttft_ms": { + "mean": 560.6182256044121, + "p50": 515.9745299897622, + "p90": 1041.3179694034625, + "p95": 1156.9083506998136, + "p99": 1249.3806557368953 + }, + "tpot_ms": { + "mean": 3.1961963242536493, + "p50": 3.206504942532288, + "p90": 4.1561452642836425, + "p95": 4.18660311174654, + "p99": 4.210969389716857 + }, + "itl_ms": { + "mean": 17.192877968244417, + "p50": 12.152664014138281, + "p90": 22.031987027730793, + "p95": 22.20277499873191, + "p99": 42.01793160755187 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_128k_0709_5_131072_4096.jsonl" + }, + { + "name": "c10_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 97.72797075999551, + "request_throughput": 0.2046497010473782, + "input_token_throughput": 28498.70900153875, + "output_token_throughput": 96.78907611035751, + "total_token_throughput": 28595.498077649107, + "total_input_tokens": 2785121, + "total_output_tokens": 9459, + "e2e_ms": { + "mean": 45387.50252430036, + "p50": 44256.56188350695, + "p90": 58910.486319282805, + "p95": 70254.48793091347, + "p99": 91123.06626138739 + }, + "ttft_ms": { + "mean": 24591.82777870301, + "p50": 29435.68563000008, + "p90": 38974.941456803936, + "p95": 39321.29660845385, + "p99": 40783.01308169611 + }, + "tpot_ms": { + "mean": 48.18920425381463, + "p50": 55.989442050476555, + "p90": 66.05037972809353, + "p95": 67.60347733728547, + "p99": 68.1321781753959 + }, + "itl_ms": { + "mean": 244.08027261090186, + "p50": 291.6475804959191, + "p90": 424.5562060939847, + "p95": 458.57615228160245, + "p99": 522.1585231143399 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_1024.jsonl" + }, + { + "name": "c10_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 110.16605063999305, + "request_throughput": 0.18154413164321512, + "input_token_throughput": 25281.118673314148, + "output_token_throughput": 23.119645164763444, + "total_token_throughput": 25304.23831847891, + "total_input_tokens": 2785121, + "total_output_tokens": 2547, + "e2e_ms": { + "mean": 48223.85499715165, + "p50": 54898.71321149985, + "p90": 75289.47817128501, + "p95": 78728.48516409722, + "p99": 80196.261309619 + }, + "ttft_ms": { + "mean": 42248.62834745145, + "p50": 49703.44070899591, + "p90": 63688.684180699056, + "p95": 64654.52864100226, + "p99": 67877.62028660624 + }, + "tpot_ms": { + "mean": 52.054832429291196, + "p50": 49.41814168657996, + "p90": 67.86011650138555, + "p95": 73.84271063763447, + "p99": 151.27544932703617 + }, + "itl_ms": { + "mean": 255.34972855337435, + "p50": 286.28537949407473, + "p90": 402.0657468063291, + "p95": 437.00676189328067, + "p99": 492.95355229522085 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_256.jsonl" + }, + { + "name": "c10_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 10, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 108.96588197600795, + "request_throughput": 0.183543689431189, + "input_token_throughput": 25559.56919261413, + "output_token_throughput": 349.9352210850334, + "total_token_throughput": 25909.504413699164, + "total_input_tokens": 2785121, + "total_output_tokens": 38131, + "e2e_ms": { + "mean": 52210.569570692314, + "p50": 50657.05102798529, + "p90": 99869.24856970436, + "p95": 100105.12522634963, + "p99": 103440.52136287354 + }, + "ttft_ms": { + "mean": 13472.463327697187, + "p50": 12262.557213485707, + "p90": 25829.017210312315, + "p95": 33426.82968994487, + "p99": 37512.71495238295 + }, + "tpot_ms": { + "mean": 20.67259007483205, + "p50": 22.335029249613676, + "p90": 33.27995795684531, + "p95": 35.936134969934166, + "p99": 44.4381474839586 + }, + "itl_ms": { + "mean": 115.51536001699138, + "p50": 16.82196301408112, + "p90": 361.40161620569415, + "p95": 415.56643948715646, + "p99": 484.59458799276035 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_10_262144_4096.jsonl" + }, + { + "name": "c2_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 16.191832354001235, + "request_throughput": 0.24703813086426518, + "input_token_throughput": 31154.905076283223, + "output_token_throughput": 55.64533897717573, + "total_token_throughput": 31210.550415260397, + "total_input_tokens": 504455, + "total_output_tokens": 901, + "e2e_ms": { + "mean": 7871.567239249998, + "p50": 7651.008348009782, + "p90": 13964.303008592105, + "p95": 14740.715870789425, + "p99": 15361.846160547282 + }, + "ttft_ms": { + "mean": 5853.5343170005945, + "p50": 5881.0410465084715, + "p90": 9820.337897987338, + "p95": 10466.903458483284, + "p99": 10984.155906880042 + }, + "tpot_ms": { + "mean": 14.357719716937018, + "p50": 3.7517795827225786, + "p90": 36.769081681037285, + "p95": 43.300111516563355, + "p99": 48.52493538498424 + }, + "itl_ms": { + "mean": 58.91917738659871, + "p50": 9.051853994606063, + "p90": 290.753513405798, + "p95": 320.6824104010593, + "p99": 350.4223257582632 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_1024.jsonl" + }, + { + "name": "c2_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 15.408737797988579, + "request_throughput": 0.25959296942038634, + "input_token_throughput": 32738.242847240246, + "output_token_throughput": 25.24541627613257, + "total_token_throughput": 32763.488263516378, + "total_input_tokens": 504455, + "total_output_tokens": 389, + "e2e_ms": { + "mean": 7672.816192993196, + "p50": 7643.010690488154, + "p90": 13423.424012202304, + "p95": 14088.425920606822, + "p99": 14620.42744733044 + }, + "ttft_ms": { + "mean": 5850.426714234345, + "p50": 5872.331345977727, + "p90": 9810.203461590574, + "p95": 10459.436945793275, + "p99": 10978.823733155441 + }, + "tpot_ms": { + "mean": 17.57577191929882, + "p50": 10.181436867008118, + "p90": 40.71390786208095, + "p95": 45.28395795219019, + "p99": 48.9399980242776 + }, + "itl_ms": { + "mean": 145.78776275971904, + "p50": 18.701617504120804, + "p90": 331.15072268410586, + "p95": 338.73794550599996, + "p99": 358.25617503229296 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_256.jsonl" + }, + { + "name": "c2_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 2, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 4 + }, + "metrics": { + "success": 4, + "failed": 0, + "duration_s": 23.653920415003086, + "request_throughput": 0.16910516015192564, + "input_token_throughput": 21326.48589110991, + "output_token_throughput": 384.41830531536493, + "total_token_throughput": 21710.904196425276, + "total_input_tokens": 504455, + "total_output_tokens": 9093, + "e2e_ms": { + "mean": 11596.579911245499, + "p50": 11369.512781500816, + "p90": 14845.453088593786, + "p95": 14994.526062793739, + "p99": 15113.784442153701 + }, + "ttft_ms": { + "mean": 5487.6301339973, + "p50": 6178.046113491291, + "p90": 8360.751068586253, + "p95": 8709.001985285431, + "p99": 8987.602718644775 + }, + "tpot_ms": { + "mean": 3.2280306182866307, + "p50": 2.0858395742806968, + "p90": 5.745412071961834, + "p95": 6.43556928685951, + "p99": 6.987695058777652 + }, + "itl_ms": { + "mean": 16.012864087155666, + "p50": 10.026946503785439, + "p90": 10.195688009844162, + "p95": 10.277631241478957, + "p99": 302.1156209943001 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_2_262144_4096.jsonl" + }, + { + "name": "c5_i262144_o1024", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 1024, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 53.372761565988185, + "request_throughput": 0.18736148751899143, + "input_token_throughput": 26116.336481420964, + "output_token_throughput": 69.32375038202683, + "total_token_throughput": 26185.66023180299, + "total_input_tokens": 1393901, + "total_output_tokens": 3700, + "e2e_ms": { + "mean": 25295.536027196795, + "p50": 22571.09521899838, + "p90": 35288.65761580818, + "p95": 38376.68780889507, + "p99": 40847.11196336459 + }, + "ttft_ms": { + "mean": 13406.144849795965, + "p50": 14497.35130299814, + "p90": 18261.543534600056, + "p95": 22995.116275803583, + "p99": 26781.97446876642 + }, + "tpot_ms": { + "mean": 38.55896514833037, + "p50": 51.21804507602598, + "p90": 56.882710194174734, + "p95": 57.18789456377013, + "p99": 57.43204205944645 + }, + "itl_ms": { + "mean": 191.76376643215096, + "p50": 268.8099675142439, + "p90": 372.18185558740515, + "p95": 414.81327724904963, + "p99": 500.33757628465537 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_1024.jsonl" + }, + { + "name": "c5_i262144_o256", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 52.2927886339894, + "request_throughput": 0.19123095671933957, + "input_token_throughput": 26655.702180204415, + "output_token_throughput": 26.695841558019804, + "total_token_throughput": 26682.398021762434, + "total_input_tokens": 1393901, + "total_output_tokens": 1396, + "e2e_ms": { + "mean": 23507.98017459747, + "p50": 25135.777328003314, + "p90": 32905.0044600066, + "p95": 35234.359566001505, + "p99": 37097.84365079744 + }, + "ttft_ms": { + "mean": 17996.009123898693, + "p50": 19306.366333999904, + "p90": 27986.28130730358, + "p95": 29425.380368152397, + "p99": 30576.659616831457 + }, + "tpot_ms": { + "mean": 41.735075029607174, + "p50": 50.43365461709989, + "p90": 58.2376680492912, + "p95": 60.423048465253345, + "p99": 62.17135279802306 + }, + "itl_ms": { + "mean": 244.9744939644976, + "p50": 286.5458000160288, + "p90": 376.96084639756003, + "p95": 426.482240395853, + "p99": 509.5562937820795 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_256.jsonl" + }, + { + "name": "c5_i262144_o4096", + "config": { + "phase": "dspark", + "concurrency": 5, + "input_len": 262144, + "output_len": 4096, + "dataset": "random", + "num_prompts": 10 + }, + "metrics": { + "success": 10, + "failed": 0, + "duration_s": 63.57839560499997, + "request_throughput": 0.15728613320361257, + "input_token_throughput": 21924.129835864875, + "output_token_throughput": 299.7873698860855, + "total_token_throughput": 22223.91720575096, + "total_input_tokens": 1393901, + "total_output_tokens": 19060, + "e2e_ms": { + "mean": 30807.30131380551, + "p50": 30501.745043002302, + "p90": 44364.00240242074, + "p95": 48506.30591822118, + "p99": 51820.14873086155 + }, + "ttft_ms": { + "mean": 10590.483214208507, + "p50": 10077.366048004478, + "p90": 16730.81445190764, + "p95": 22250.94545695318, + "p99": 26667.050260989636 + }, + "tpot_ms": { + "mean": 17.46113578915578, + "p50": 10.161597631207403, + "p90": 39.99704357805789, + "p95": 45.408594531434666, + "p99": 49.737835294136104 + }, + "itl_ms": { + "mean": 63.039536119419395, + "p50": 21.629737981129438, + "p90": 288.61644080025144, + "p95": 342.8714099165518, + "p99": 451.42972400062735 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_256k_0709_5_262144_4096.jsonl" + }, + { + "name": "c25_i65536_o1024", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 62.70547563501168, + "request_throughput": 0.79737853024246, + "input_token_throughput": 29366.669837870166, + "output_token_throughput": 412.73907482410215, + "total_token_throughput": 29779.408912694267, + "total_input_tokens": 1841451, + "total_output_tokens": 25881, + "e2e_ms": { + "mean": 29981.59460285853, + "p50": 29794.13296599523, + "p90": 48801.88996298821, + "p95": 57554.651349992484, + "p99": 60995.55153899796 + }, + "ttft_ms": { + "mean": 11692.068987198872, + "p50": 9282.525921502383, + "p90": 23589.7407762066, + "p95": 26078.115880547557, + "p99": 29135.598926548842 + }, + "tpot_ms": { + "mean": 42.82228902198422, + "p50": 45.10009716195819, + "p90": 53.573378286077535, + "p95": 56.18558621741822, + "p99": 133.88920998555187 + }, + "itl_ms": { + "mean": 197.6815361781092, + "p50": 264.1833170055179, + "p90": 296.874576000846, + "p95": 313.95786673238035, + "p99": 335.2813712626812 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_1024.jsonl" + }, + { + "name": "c25_i65536_o256", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 59.856636811979115, + "request_throughput": 0.8353292577573201, + "input_token_throughput": 30764.3579405295, + "output_token_throughput": 98.78603802238068, + "total_token_throughput": 30863.143978551878, + "total_input_tokens": 1841451, + "total_output_tokens": 5913, + "e2e_ms": { + "mean": 24957.478126779897, + "p50": 26267.052594514098, + "p90": 34792.27077438264, + "p95": 37092.857118272506, + "p99": 39437.04205935879 + }, + "ttft_ms": { + "mean": 19438.036874281825, + "p50": 23692.236121496535, + "p90": 27105.122378107626, + "p95": 27635.110721291854, + "p99": 29212.481069455676 + }, + "tpot_ms": { + "mean": 51.68095443735579, + "p50": 49.096686958535194, + "p90": 58.59885411730775, + "p95": 65.5217898325625, + "p99": 134.9373555736381 + }, + "itl_ms": { + "mean": 251.79774711594382, + "p50": 267.6665144826984, + "p90": 299.91588299162686, + "p95": 320.7846462682937, + "p99": 347.9563715533004 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_256.jsonl" + }, + { + "name": "c25_i65536_o4096", + "config": { + "phase": "dspark", + "concurrency": 25, + "input_len": 65536, + "output_len": 4096, + "dataset": "random", + "num_prompts": 50 + }, + "metrics": { + "success": 50, + "failed": 0, + "duration_s": 82.31095573300263, + "request_throughput": 0.6074525505716181, + "input_token_throughput": 22371.882134053136, + "output_token_throughput": 1409.204873969074, + "total_token_throughput": 23781.08700802221, + "total_input_tokens": 1841451, + "total_output_tokens": 115993, + "e2e_ms": { + "mean": 38164.69241847808, + "p50": 36029.353522491874, + "p90": 68485.24556439371, + "p95": 72245.56214315962, + "p99": 73496.425884809 + }, + "ttft_ms": { + "mean": 8084.046624338953, + "p50": 3264.411232987186, + "p90": 23582.51729948097, + "p95": 25048.60101691301, + "p99": 29143.3001965165 + }, + "tpot_ms": { + "mean": 14.71954587074255, + "p50": 13.270791017156972, + "p90": 22.879623646797487, + "p95": 30.574523231211494, + "p99": 42.06810737029239 + }, + "itl_ms": { + "mean": 72.87677341734114, + "p50": 23.865676994319074, + "p90": 272.30329640733544, + "p95": 286.6157213851693, + "p99": 309.7392923911684 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/yy/experiments/dsv4_h200_long_context_matrix/results/20260709-020227/dspark/raw_outputs/vllm_dspark_64k_0709_25_65536_4096.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_long_context_matrix/run_bench.sh b/experiments/dsv4_h200_long_context_matrix/run_bench.sh index 5f9dc6f..fa8c89e 100755 --- a/experiments/dsv4_h200_long_context_matrix/run_bench.sh +++ b/experiments/dsv4_h200_long_context_matrix/run_bench.sh @@ -14,6 +14,13 @@ source "${SCRIPT_DIR}/config.env" RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" RESULT_BASE="${SCRIPT_DIR}/results" +# Comma-separated list of backends to run. Default: sglang,vllm,dspark. +BACKENDS="${BACKENDS:-sglang,vllm,dspark}" + +backend_enabled() { + local backend="$1" + [[ ",${BACKENDS}," == *",${backend},"* ]] +} log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" mkdir -p "$log_dir_global" @@ -52,6 +59,8 @@ stop_server() { # Fallback cleanup. if [[ "$backend" == "sglang" ]]; then pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true + elif [[ "$backend" == "dspark" ]]; then + pkill -9 -f "vllm serve.*DeepSeek-V4-Flash-DSpark" 2>/dev/null || true else pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true fi @@ -62,13 +71,16 @@ start_server() { local backend="$1" local max_model_len="$2" local max_slots="$3" - local start_script + local start_script port if [[ "$backend" == "sglang" ]]; then start_script="${SGLANG_START_SCRIPT}" - local port="$SGLANG_PORT" + port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + start_script="${DSPARK_START_SCRIPT}" + port="$DSPARK_PORT" else start_script="${VLLM_START_SCRIPT}" - local port="$VLLM_PORT" + port="$VLLM_PORT" fi log "starting ${backend} server (max_model_len=${max_model_len}, slots=${max_slots}) with ${start_script}" @@ -88,13 +100,15 @@ run_warmup() { local port if [[ "$backend" == "sglang" ]]; then port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" else port="$VLLM_PORT" fi log "warming up ${backend} (input=${input_len}, output=${output_len}, num=1)" "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ - --backend "$backend" \ + --backend vllm \ --port "$port" \ --input-len "$input_len" \ --output-len "$output_len" \ @@ -142,6 +156,8 @@ run_group() { local port if [[ "$backend" == "sglang" ]]; then port="$SGLANG_PORT" + elif [[ "$backend" == "dspark" ]]; then + port="$DSPARK_PORT" else port="$VLLM_PORT" fi @@ -161,9 +177,22 @@ run_group() { warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')" run_warmup "$backend" "$warmup_input_len" 256 + # Bench client backend: DSpark is served by vLLM API, so use vllm client. + local client_backend="$backend" + if [[ "$backend" == "dspark" ]]; then + client_backend="vllm" + fi + + # Output filename prefix: DSpark raw files must start with "vllm_" so that + # scripts/common/parse_backend.py (which only knows sglang/vllm) can parse them. + local file_prefix="${backend}" + if [[ "$backend" == "dspark" ]]; then + file_prefix="vllm_dspark" + fi + for scenario in "${scenarios_ref[@]}"; do read -r concurrency input_len output_len num_prompts <<< "$scenario" - output_file="${raw_dir}/${backend}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + output_file="${raw_dir}/${file_prefix}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" detail_log="${phase_log_dir}/${backend}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log" if scenario_already_completed "$output_file" "$num_prompts"; then @@ -174,7 +203,7 @@ run_group() { log "running ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ - --backend "$backend" \ + --backend "$client_backend" \ --host 127.0.0.1 \ --port "$port" \ --dataset-name random \ @@ -200,8 +229,13 @@ run_group() { parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" - log "parsing ${backend} results in ${result_root}" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ + # parse_backend.py only knows sglang/vllm; DSpark raw files use vllm client format. + local parser_backend="$backend" + if [[ "$backend" == "dspark" ]]; then + parser_backend="vllm" + fi + log "parsing ${backend} results in ${result_root} (parser_backend=${parser_backend})" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$parser_backend" \ >> "${result_root}/logs/parse.log" 2>&1 || { log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" } @@ -213,20 +247,28 @@ write_backend_metadata() { ensure_result_root "$result_root" local meta_json="${result_root}/results.json" - local env_path + local env_path model_path engine_name if [[ "$backend" == "sglang" ]]; then env_path="$VENV_SGLANG" + model_path="$MODEL_PATH" + engine_name="sglang" + elif [[ "$backend" == "dspark" ]]; then + env_path="$VENV_VLLM_DSPARK" + model_path="$DSPARK_MODEL_PATH" + engine_name="vllm-dspark" else env_path="$VENV_VLLM" + model_path="$MODEL_PATH" + engine_name="vllm" fi write_metadata_json \ "$meta_json" \ "${EXPERIMENT_NAME}_${backend}" \ "$RUN_ID" \ - "$MODEL_PATH" \ - "$backend" \ - "$backend" \ + "$model_path" \ + "${backend}" \ + "$engine_name" \ "$HARDWARE" \ "$ACCELERATOR" \ "$CHIP" \ @@ -254,11 +296,13 @@ write_backend_metadata() { # --------------------------------------------------------------------------- # Cleanup any leftovers. -stop_server sglang -stop_server vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && stop_server "$backend" +done -write_backend_metadata sglang -write_backend_metadata vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && write_backend_metadata "$backend" +done for group in "${CONTEXT_GROUPS[@]}"; do read -r group_label max_model_len max_slots <<< "$group" @@ -270,24 +314,28 @@ for group in "${CONTEXT_GROUPS[@]}"; do continue fi - # Run SGLang for this group. - run_group sglang "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" - - # Run vLLM for this group. - run_group vllm "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" + for backend in sglang vllm dspark; do + backend_enabled "$backend" && run_group "$backend" "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name" + done done # Parse and compare. -parse_backend sglang -parse_backend vllm +for backend in sglang vllm dspark; do + backend_enabled "$backend" && parse_backend "$backend" +done -log "generating comparison report" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ - --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ - --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ - --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ - >> "${log_dir_global}/compare.log" 2>&1 || { - log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" - } +if backend_enabled sglang && backend_enabled vllm && backend_enabled dspark; then + log "generating three-way comparison report" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ + --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ + --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ + --dspark "${RESULT_BASE}/${RUN_ID}/dspark" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } +else + log "skipping three-way comparison (not all backends enabled: BACKENDS=${BACKENDS})" +fi log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_h200_long_context_matrix/start_dspark.sh b/experiments/dsv4_h200_long_context_matrix/start_dspark.sh new file mode 100755 index 0000000..6c9ee56 --- /dev/null +++ b/experiments/dsv4_h200_long_context_matrix/start_dspark.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Start vLLM with DSpark speculative decoding for the long-context matrix. +# Usage: start_dspark.sh +set -e + +MAX_MODEL_LEN="${1:-80000}" +MAX_NUM_SEQS="${2:-64}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs tmp + +VENV="${VENV_VLLM_DSPARK}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/dsv4_h200_long_context_matrix_dspark_${MAX_MODEL_LEN}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/dsv4_h200_long_context_matrix_dspark.pid" + +rm -f "$PID_FILE" + +echo "=== Starting DeepSeek-V4-Flash-DSpark server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_num_seqs=$MAX_NUM_SEQS) ===" +echo "Model: $DSPARK_MODEL_PATH" +echo "Port: $DSPARK_PORT" +echo "Log: $LOG" + +nohup vllm serve "$DSPARK_MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --spec-method dspark \ + --spec-model "$DSPARK_MODEL_PATH" \ + --spec-tokens 5 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$DSPARK_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${DSPARK_PORT}/health" >/dev/null 2>&1; then + echo "DSpark server is ready at http://127.0.0.1:${DSPARK_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: DSpark server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: DSpark server not healthy after 240 retries" +tail -200 "$LOG" +exit 1