From a7e2037471541e317e533a7c8a81c2a9737f5256 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Mon, 20 Jul 2026 04:27:07 +0000 Subject: [PATCH] feat(p800): add sglang profiling experiment with PROFILE_REPORT - New experiment: dsv4_p800_sglang_profile for PyTorch profiler testing - Documents XPU cuptiActivityDisable bug (error 17) when saving traces - Includes benchmark results (ISL=4k, OSL=1k, C=16, TP8/DP1) - PROFILE_REPORT.md with full analysis of P800 vs H20 performance gap - Add dsv4_p800_sglang_tp_dp_official experiment config --- .../p800/dsv4_p800_sglang_profile/.gitignore | 5 + .../PROFILE_REPORT.md | 299 +++ .../p800/dsv4_p800_sglang_profile/config.env | 39 + .../patches/bench_serving.py | 2366 +++++++++++++++++ .../dsv4_p800_sglang_profile/run_profile.sh | 143 + .../start_sglang_docker.sh | 171 ++ .../dsv4_p800_sglang_tp_dp_official/README.md | 83 + .../adaptive_config.env | 58 + .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../run_manifest.json | 27 + .../adaptive_20260717-072013/shapes.tsv | 25 + .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 + .../adaptive_20260717-072159/shapes.tsv | 25 + .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../run_manifest.json | 27 + .../adaptive_20260717-073006/shapes.tsv | 25 + .../adaptive_points.jsonl | 0 .../adaptive_shapes.jsonl | 0 .../adaptive_summary.jsonl | 0 .../adaptive_summary.md | 6 + .../run_manifest.json | 27 + .../adaptive_20260717-075355/shapes.tsv | 25 + .../adaptive_points.jsonl | 99 + .../adaptive_shapes.jsonl | 36 + .../adaptive_summary.jsonl | 36 + .../adaptive_summary.md | 42 + .../run_manifest.json | 27 + .../adaptive_20260717-075359/shapes.tsv | 25 + .../adaptive_points.jsonl | 2 + .../adaptive_shapes.jsonl | 1 + .../adaptive_summary.jsonl | 1 + .../smoke-20260717-072308/adaptive_summary.md | 7 + .../smoke-20260717-072308/run_manifest.json | 27 + .../smoke-20260717-072308/shapes.tsv | 25 + .../compare.py | 162 ++ .../config.env | 102 + .../generate_scenarios.py | 98 + .../matrix.json | 54 + .../patches/bench_serving.py | 2366 +++++++++++++++++ .../run_adaptive_concurrency.sh | 153 ++ .../run_adaptive_concurrency_add16.sh | 162 ++ .../run_bench.sh | 559 ++++ .../start_sglang_docker.sh | 183 ++ .../start_sglang_dp.sh | 11 + 50 files changed, 7562 insertions(+) create mode 100644 experiments/p800/dsv4_p800_sglang_profile/.gitignore create mode 100644 experiments/p800/dsv4_p800_sglang_profile/PROFILE_REPORT.md create mode 100644 experiments/p800/dsv4_p800_sglang_profile/config.env create mode 100644 experiments/p800/dsv4_p800_sglang_profile/patches/bench_serving.py create mode 100755 experiments/p800/dsv4_p800_sglang_profile/run_profile.sh create mode 100755 experiments/p800/dsv4_p800_sglang_profile/start_sglang_docker.sh create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/README.md create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_config.env create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.md create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.md create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.md create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_points.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_shapes.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.jsonl create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.md create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/run_manifest.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/shapes.tsv create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/compare.py create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/generate_scenarios.py create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json create mode 100644 experiments/p800/dsv4_p800_sglang_tp_dp_official/patches/bench_serving.py create mode 100755 experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh create mode 100755 experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh create mode 100755 experiments/p800/dsv4_p800_sglang_tp_dp_official/run_bench.sh create mode 100755 experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh create mode 100755 experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_dp.sh diff --git a/experiments/p800/dsv4_p800_sglang_profile/.gitignore b/experiments/p800/dsv4_p800_sglang_profile/.gitignore new file mode 100644 index 0000000..2a24910 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/.gitignore @@ -0,0 +1,5 @@ +runtime/ +profile_output/ +bench_output/ +*.log +*.csv diff --git a/experiments/p800/dsv4_p800_sglang_profile/PROFILE_REPORT.md b/experiments/p800/dsv4_p800_sglang_profile/PROFILE_REPORT.md new file mode 100644 index 0000000..32f7cd4 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/PROFILE_REPORT.md @@ -0,0 +1,299 @@ +# P800 SGLang Profiling 报告 + +> 日期: 2026-07-20 +> 实验目录: `/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_profile/` +> 测试场景: ISL=4096, OSL=1024, Concurrency=16, TP=8, DP=1 + +--- + +## 一、实验目的 + +定位 P800 SGLang 与 H20 SGLang 在 DeepSeek-V4-Flash-INT8 模型推理上的性能差距根因。 + +**已知差距**（TP8/DP1, ISL=4k, OSL=1k, C=16）： + +| 指标 | P800 | H20 | 差距 | +|:----|:---:|:---:|:----:| +| Total TPS | 1,863 | 4,866 | H20 快 2.6x | +| TPOT P50 | 37ms | 13ms | H20 快 2.8x | +| ITL P50 | 35ms | 11ms | H20 快 3.1x | +| E2E P95 | 49s | 17s | H20 快 2.9x | + +--- + +## 二、实验环境 + +### 2.1 硬件 + +| 项目 | 配置 | +|:----|:----| +| 服务器 | p800.2 (gpu049) | +| XPU | 8× Kunlun P800 XPU, 96GB/XPU | +| 驱动版本 | 515.58 | + +### 2.2 软件 + +| 项目 | 版本 | +|:----|:----| +| Docker 镜像 | `iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202` | +| SGLang | 容器内预装版本 | +| PyTorch | 容器内预装版本 | +| 模型 | DeepSeek-V4-Flash-INT8 (INT8 量化) | +| 推理引擎 | SGLang (XPU 定制版) | + +### 2.3 服务器启动参数 + +``` +--host 0.0.0.0 --port 30015 --model-path /models +--attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa +--trust-remote-code --disable-custom-all-reduce --page-size 64 +--mem-fraction-static 0.8 --tensor-parallel-size 8 +--disable-shared-experts-fusion --quantization w8a8_int8 +--kv-cache-dtype float16 --disable-piecewise-cuda-graph +--cuda-graph-max-bs 32 --watchdog-timeout 3000000 +--tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 +--constrained-json-disable-any-whitespace +--enable-metrics --enable-request-time-stats-logging +--context-length 140000 +``` + +### 2.4 关键 Patch + +容器启动时挂载了以下 PATCH 文件（位于 `/data1/yy/sskj/platforms/patches/kunlun_p800/`）： + +| Patch 文件 | 作用 | +|:----------|:-----| +| `hf_transformers_utils.py.patched` | 修复 DeepSeek-V4 模型架构识别 | +| `fp8_utils.py.patched` | FP8 量化工具修复 | +| `config_backup_small_w8a8_int8.json` | INT8 配置备份 | +| `parallel_state.py` | 分布式状态修复 | +| `vocab_parallel_embedding.py` | Embedding 并行修复 | +| `sitecustomize_xpu.py` | XPU 环境初始化 | +| `nic_priority_matrix_test.json` | 网络拓扑配置 | +| `bench_serving.py` (patched) | 增加 P95 TTFT/TPOT/E2E 指标输出 | + +--- + +## 三、实验目录结构 + +``` +/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_profile/ +├── config.env # 配置文件 +├── start_sglang_docker.sh # 容器启动脚本 +├── run_profile.sh # 一键 profile 运行脚本 +├── patches/ +│ └── bench_serving.py # 已打补丁的 bench_serving +├── profile_output/ # Profile 输出目录 +│ ├── / +│ │ ├── xpu_monitor.csv # XPU 利用率监控 +│ │ └── bench_serving.log # Benchmark 日志 +│ └── ... +├── bench_output/ # Benchmark 结果目录 +│ └── results_no_profile.jsonl # 基准测试结果 +└── runtime/ + ├── logs/ # 服务器日志 + └── entrypoint_*.sh # 容器启动入口脚本 +``` + +--- + +## 四、PyTorch Profiler Bug + +### 4.1 Bug 描述 + +SGLang bench_serving 的 `--profile` 功能使用 PyTorch Profiler 采集 GPU/XPU 的 kernel 执行 trace。在 P800 XPU 上，profiler 可以**启动并采集数据**，但在**停止保存 trace 时崩溃**，导致服务器进程被 kill。 + +### 4.2 错误信息 + +``` +RuntimeError: [FATAL] +[/mnt/ssd3/xpytorch/workspace/.../cupti/cupti_activity.cpp:137:cuptiActivityDisable] +CUTOOL_CALL func pExportTable->etiEndEventSampling(ctx) failed with error 17 +``` + +### 4.3 触发条件 + +- 使用 `--profile` 或 `--profile --profile-num-steps N` 启动 profiler +- Profiler 运行 N 步后尝试自动停止，或 benchmark 完成后客户端发送 stop_profile 请求 +- 在 `cuptiActivityDisable()` 调用时崩溃 + +### 4.4 影响 + +- ❌ Chrome Trace JSON 文件无法生成 +- ❌ 服务器进程被 kill（SIGKILL） +- ✅ 不影响非 profiler 模式的正常运行 + +### 4.5 根因 + +XPU 的 CUPTI（CUDA Profiling Tools Interface）实现在 `etiEndEventSampling` 调用时存在 bug，无法正确停止事件采样。这是 XPU 驱动/PyTorch XPU 后端的已知问题。 + +### 4.6 临时解决方案 + +1. **不使用 PyTorch Profiler**，改用 SGLang 内置的 `--enable-metrics` 和 `--enable-request-time-stats-logging` 获取聚合指标 +2. 使用 `xpu-smi` 实时监控 XPU 利用率、显存、功耗 +3. 使用 `--output-details` 获取每个请求的延迟分布 +4. 在 H20 上使用同样的 profiler（NVIDIA 版本无此 bug）获取 Chrome Trace 做对比 + +--- + +## 五、测试过程记录 + +### 5.1 创建实验目录 + +```bash +# 创建目录 +mkdir -p /data1/yy/sskj/experiments/p800/dsv4_p800_sglang_profile/patches + +# 复制 patched bench_serving.py +cp /data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_matrix/patches/bench_serving.py \ + /data1/yy/sskj/experiments/p800/dsv4_p800_sglang_profile/patches/bench_serving.py + +# 创建配置文件 +# config.env: 基于 dsv4_p800_sglang_tp_dp_matrix 的配置，增加 PROFILE_OUTPUT_DIR 等 +``` + +### 5.2 启动脚本要点 + +`start_sglang_docker.sh` 需要包含： + +1. 所有必要的环境变量（`XPU_VISIBLE_DEVICES`, `SGLANG_DSV4_MODE` 等 30+ 个） +2. 所有必要的 Patch 挂载（8 个关键 patch 文件） +3. `XPU_ENABLE_PROFILER_TRACING=1` 环境变量（启用 profiler 的必要条件） +4. Profile 输出目录的挂载（`-v host_dir:/workspace/profile_output:rw`） +5. 使用独立容器名，避免与其他实验冲突 + +### 5.3 启动命令 + +```bash +# 方式1：一键运行 +bash run_profile.sh + +# 方式2：分步执行 +bash start_sglang_docker.sh 8 1 # 启动服务器 +docker exec ... sglang.bench_serving ... # 运行 benchmark +docker rm -f sglang-dsv4-flash-profile # 停止服务器 +``` + +### 5.4 服务器启动耗时 + +从 `docker run` 到 `health check` 通过约需 **2分40秒**。 + +### 5.5 Benchmark 执行 + +```bash +docker exec sglang-dsv4-flash-profile \ + /root/miniconda/envs/python310_torch25_cuda/bin/python \ + -m sglang.bench_serving \ + --backend sglang --host 127.0.0.1 --port 30015 \ + --model /data1/models/DeepSeek-V4-Flash-INT8 \ + --dataset-name random --random-input-len 4096 \ + --random-output-len 1024 --random-range-ratio 1.0 \ + --num-prompts 80 --max-concurrency 16 --request-rate 10000 \ + --output-file /workspace/bench_output/results.jsonl --output-details +``` + +### 5.6 Profiler 调用（已确认不可用） + +```bash +# 以下参数会导致 profiler 崩溃 +--profile --profile-num-steps 30 \ +--profile-output-dir /workspace/profile_output \ +--profile-prefix p800_4k1k_c16 +``` + +--- + +## 六、基准测试结果 + +### 6.1 本次测试结果 + +| 指标 | 数值 | +|:----|:----:| +| Successful requests | 80 | +| Benchmark duration (s) | 250.70 | +| Total input tokens | 327,680 | +| Total generated tokens | 81,920 | +| Request throughput (req/s) | 0.32 | +| Input token throughput (tok/s) | 1,306.97 | +| Output token throughput (tok/s) | 326.74 | +| Peak output token throughput (tok/s) | 464.00 | +| Peak concurrent requests | 32 | +| **Total token throughput (tok/s)** | **1,633.71** | +| Concurrency | 16.00 | + +### 6.2 延迟分布 + +| 指标 | P50 | P95 | P99 | +|:----|:--:|:--:|:--:| +| E2E Latency (ms) | 49,422 | 53,409 | 53,410 | +| **TTFT (ms)** | **9,335** | **14,336** | **15,223** | +| **TPOT (ms)** | **40.4** | **45.9** | **46.4** | +| **ITL (ms)** | **34.9** | **53.4** | **58.4** | +| Max ITL (ms) | 9,895 | | | + +### 6.3 XPU 监控 + +| 指标 | 数值 | +|:----|:----:| +| XPU 利用率 P50 | 95% | +| 显存使用率 | 98.5% (96GB/XPU 几乎占满) | + +--- + +## 七、与 H20 的对比分析 + +### 7.1 同配置对比（TP8/DP1, C=16, ISL=4k, OSL=1k） + +| 指标 | P800 | H20 | 差距 | 瓶颈类型 | +|:----|:---:|:---:|:----:|:--------:| +| Total TPS | 1,634 | 4,866 | 3.0x | 综合 | +| TTFT P95 | 14,336ms | 5,341ms | 2.7x | Prefill (计算) | +| TPOT P50 | 40.4ms | 13.1ms | **3.1x** | **Decode (内存带宽)** | +| ITL P50 | 34.9ms | 11.2ms | 3.1x | Decode (内存带宽) | +| ITL P95 | 53.4ms | 11.5ms | 4.6x | Decode 稳定性 | +| ITL Max | 9,895ms | — | — | 调度抖动 | + +### 7.2 核心瓶颈分析 + +**Decode 阶段（TPOT/ITL）是最大差距所在**，差距约 3.1x。即使在没有并发竞争的 C=1 场景下，P800 的 TPOT 也高达 26.6ms，而 H20 仅 9.1ms。 + +**根本原因：** + +1. **内存带宽不足**（最主要） + - LLM decode 是 memory-bound 操作 + - P800 XPU 的 HBM 带宽可能远低于 H20 的 4.0 TB/s HBM3 + - 即使 INT8 量化减少了一半数据量，带宽仍然不够 + +2. **Kernel 效率低** + - NSA attention 等定制 kernel 在 XPU 上的优化程度不如 CUDA + - ITL 波动大（P95=53.4ms vs P50=34.9ms），说明 kernel 执行不稳定 + +3. **通信开销大** + - TP=8 需要 8 张 XPU 做 all-reduce + - XPULINK 互联带宽可能低于 NVLink + +4. **显存瓶颈** + - 显存使用率 98.5%，几乎没有余量 + - 限制了 batch size 和内存管理效率 + +--- + +## 八、实验文件清单 + +| 文件 | 路径 | 说明 | +|:----|:----|:-----| +| config.env | `dsv4_p800_sglang_profile/config.env` | 实验配置 | +| start_sglang_docker.sh | `dsv4_p800_sglang_profile/start_sglang_docker.sh` | 容器启动脚本 | +| run_profile.sh | `dsv4_p800_sglang_profile/run_profile.sh` | 一键运行脚本 | +| bench_serving.py (patched) | `dsv4_p800_sglang_profile/patches/bench_serving.py` | 打补丁的 bench_serving | +| 基准测试结果 | `dsv4_p800_sglang_profile/bench_output/results_no_profile.jsonl` | 80请求的benchmark结果 | +| 服务器日志 | `dsv4_p800_sglang_profile/runtime/logs/` | 服务器启动日志 | + +--- + +## 九、后续建议 + +1. **在 H20 上跑同样的 profile**（使用 `--profile --profile-num-steps 30`），获取 Chrome Trace 文件，与 P800 做 kernel 级别的对比 +2. **降低 TP 度数**：尝试 TP4/DP2 代替 TP8/DP1，减少通信开销 +3. **升级 XPU 驱动**：修复 `cuptiActivityDisable` bug 后重新尝试 PyTorch Profiler +4. **添加手动计时**：在 SGLang 的 decode loop 中添加分段计时（attention vs MoE vs all-reduce），绕过 profiler 的限制 \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_profile/config.env b/experiments/p800/dsv4_p800_sglang_profile/config.env new file mode 100644 index 0000000..470f9b3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/config.env @@ -0,0 +1,39 @@ +# Profile experiment for DeepSeek-V4-Flash-INT8 on Kunlun P800 (8 XPUs) +# using SGLang-XPU. Runs a single scenario with PyTorch profiler enabled. +EXPERIMENT="dsv4_p800_sglang_profile" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +SGLANG_PORT="${SGLANG_PORT:-30015}" + +# Dedicated container name so this experiment never touches other containers. +CONTAINER_NAME="sglang-dsv4-flash-profile" + +# Python interpreter inside the P800 SGLang container. +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +export XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs and temp files. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# SGLang server settings +CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" + +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" + +# Profiler output directory (host path, mounted into container) +PROFILE_OUTPUT_DIR="${PROFILE_OUTPUT_DIR:-${SCRIPT_DIR}/profile_output}" + +# ShareGPT seed dataset +DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Benchmark output dir (host path, mounted into container) +BENCH_OUTPUT_DIR="${BENCH_OUTPUT_DIR:-${SCRIPT_DIR}/bench_output}" + +# Dry-run mode +DRY_RUN="${DRY_RUN:-0}" diff --git a/experiments/p800/dsv4_p800_sglang_profile/patches/bench_serving.py b/experiments/p800/dsv4_p800_sglang_profile/patches/bench_serving.py new file mode 100644 index 0000000..97c752f --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/patches/bench_serving.py @@ -0,0 +1,2366 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +""" + +import argparse +import asyncio +import copy +import importlib.util +import json +import os +import random +import shutil +import sys +import time +import traceback +import uuid +import warnings +from argparse import ArgumentParser +from copy import deepcopy +from dataclasses import dataclass, field, replace +from datetime import datetime +from pathlib import Path +from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Union + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +from sglang.benchmark.datasets import DatasetRow, get_dataset +from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time +from sglang.benchmark.utils import ( + get_tokenizer, + parse_custom_headers, + remove_prefix, + set_ulimit, +) +from sglang.srt.utils.network import NetworkAddress + +_ROUTING_KEY_HEADER = "X-SMG-Routing-Key" + +_EMBEDDING_UNSUPPORTED_DATASETS = {"image", "mmmu", "mooncake"} + +TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and ( + shutil.which("gnuplot") is not None +) + +global args + + +# don't want to import sglang package here +def _get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def _create_bench_client_session(): + # When the pressure is big, the read buffer could be full before aio thread read + # the content. We increase the read_bufsize from 64K to 10M. + # Define constants for timeout and buffer size for clarity and maintainability + BENCH_AIOHTTP_TIMEOUT_SECONDS = 6 * 60 * 60 # 6 hours + BENCH_AIOHTTP_READ_BUFSIZE_BYTES = 10 * 1024**2 # 10 MB + + aiohttp_timeout = aiohttp.ClientTimeout(total=BENCH_AIOHTTP_TIMEOUT_SECONDS) + return aiohttp.ClientSession( + timeout=aiohttp_timeout, read_bufsize=BENCH_AIOHTTP_READ_BUFSIZE_BYTES + ) + + +@dataclass +class RequestFuncInput: + prompt: Union[str, List[str], List[Dict[str, str]]] + api_url: str + prompt_len: int + output_len: int + model: str + lora_name: str + image_data: Optional[List[str]] + extra_request_body: Dict[str, Any] + timestamp: Optional[float] = None + routing_key: Optional[str] = None + + +@dataclass +class RequestFuncOutput: + generated_text: str = "" + success: bool = False + latency: float = 0.0 + ttft: float = 0.0 # Time to first token + itl: List[float] = field(default_factory=list) # List of inter-token latencies + text_chunks: List[str] = field(default_factory=list) + prompt_len: int = 0 + error: str = "" + output_len: int = 0 + start_time: float = 0.0 + + @staticmethod + def init_new(request_func_input: RequestFuncInput): + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + return output + + +def get_auth_headers() -> Dict[str, str]: + openai_api_key = os.environ.get("OPENAI_API_KEY") + if openai_api_key: + return {"Authorization": f"Bearer {openai_api_key}"} + else: + api_key = os.environ.get("API_KEY") + if api_key: + return {"Authorization": f"{api_key}"} + return {} + + +def get_request_headers() -> Dict[str, str]: + headers = get_auth_headers() + if h := getattr(args, "header", None): + headers.update(parse_custom_headers(h)) + return headers + + +def wait_for_endpoint(url: str, timeout_sec: int = 60) -> bool: + """Wait for the server to become ready by polling the given URL.""" + print(f"Waiting up to {timeout_sec}s for {url} to become ready...") + start_time = time.perf_counter() + headers = get_auth_headers() + while True: + try: + response = requests.get(url, headers=headers, timeout=5) + if response.status_code == 200: + elapsed = time.perf_counter() - start_time + print(f"Server ready in {elapsed:.1f}s.") + return True + except requests.exceptions.RequestException: + pass + elapsed = time.perf_counter() - start_time + if elapsed >= timeout_sec: + print(f"Server did not become ready within {timeout_sec}s timeout.") + return False + time.sleep(1) + + +# trt llm does not support ignore_eos +# https://github.com/triton-inference-server/tensorrtllm_backend/issues/505 +async def async_request_trt_llm( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith("generate_stream") + + async with _create_bench_client_session() as session: + payload = { + "accumulate_tokens": True, + "text_input": request_func_input.prompt, + "temperature": 0.000001, + "top_p": 1.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "min_length": request_func_input.output_len, + "end_id": 1048576, + **request_func_input.extra_request_body, + } + if args.disable_ignore_eos: + del payload["min_length"] + del payload["end_id"] + output = RequestFuncOutput.init_new(request_func_input) + + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + + data = json.loads(chunk) + output.generated_text += data["text_output"] + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + + output.latency = most_recent_timestamp - st + output.success = True + output.output_len = request_func_input.output_len + + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + # Build payload with defaults that can be overridden by extra_request_body + payload = { + "model": request_func_input.model, + "prompt": prompt, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + } + + # Add temperature default only if not specified in extra_request_body + if "temperature" not in request_func_input.extra_request_body: + payload["temperature"] = 0.0 + + # Add ignore_eos default only if not specified in extra_request_body + if "ignore_eos" not in request_func_input.extra_request_body: + payload["ignore_eos"] = not args.disable_ignore_eos + + if args.return_logprob and args.top_logprobs_num > 0: + payload["logprobs"] = args.top_logprobs_num + + # Merge in extra parameters - these will override defaults if present + payload.update(request_func_input.extra_request_body) + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + if request_func_input.image_data: + payload.update({"image_data": request_func_input.image_data}) + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append( + data["choices"][0]["text"] + ) + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_chat_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + """Makes a request to the OpenAI Chat Completions API. + + Handles both streaming and non-streaming responses, including support + for image data in messages. Calculates and returns various performance + metrics. + + Args: + request_func_input: Input parameters for the request. + pbar: Optional tqdm progress bar to update. + + Returns: + RequestFuncOutput: Output of the request, including generated text, + latency, TTFT, ITL, and success status. + """ + api_url = request_func_input.api_url + assert api_url.endswith( + "chat/completions" + ), "OpenAI Chat Completions API URL must end with 'chat/completions'." + + # TODO put it to other functions when `pbar` logic is refactored + if getattr(args, "print_requests", False): + rid = str(uuid.uuid4()) + input_partial = deepcopy(request_func_input) + input_partial.prompt = "..." + request_start_time = time.time() + print( + f'rid={rid} time={request_start_time} message="request start" request_func_input="{str(input_partial)}"' + ) + + if isinstance(request_func_input.prompt, list): + messages = request_func_input.prompt + elif request_func_input.image_data: + # Build multi-image content: a list of image_url entries followed by the text + content_items = [ + { + "type": "image_url", + "image_url": {"url": img_url}, + } + for img_url in request_func_input.image_data + ] + content_items.append({"type": "text", "text": request_func_input.prompt}) + messages = [ + { + "role": "user", + "content": content_items, + }, + ] + else: + messages = [{"role": "user", "content": request_func_input.prompt}] + + async with _create_bench_client_session() as session: + # Build payload with defaults that can be overridden by extra_request_body + payload = { + "model": request_func_input.model, + "messages": messages, + "max_completion_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + } + + # Add temperature default only if not specified in extra_request_body + if "temperature" not in request_func_input.extra_request_body: + payload["temperature"] = 0.0 + + # Add ignore_eos default only if not specified in extra_request_body + # Default to False for more realistic behavior (respect EOS tokens) + if "ignore_eos" not in request_func_input.extra_request_body: + payload["ignore_eos"] = not args.disable_ignore_eos + + # Merge in extra parameters (tools, temperature, top_p, etc.) + # These will override defaults if present + payload.update(request_func_input.extra_request_body) + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + if args.disable_stream: + # Non-streaming response + response_json = await response.json() + output.generated_text = response_json["choices"][0]["message"][ + "content" + ] + output.success = True + output.latency = time.perf_counter() - st + output.ttft = ( + output.latency + ) # For non-streaming, TTFT = total latency + output.output_len = response_json.get("usage", {}).get( + "completion_tokens", output_len + ) + else: + # Streaming response + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # Check if this chunk contains content + delta = data.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + + if content: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append(content) + output.itl.append( + timestamp - most_recent_timestamp + ) + + most_recent_timestamp = timestamp + generated_text += content + + # Check for usage info in final chunk + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + # TODO put it to other functions when `pbar` logic is refactored + if getattr(args, "print_requests", False): + curr_t = time.time() + output_partial = deepcopy(output) + output_partial.generated_text = "..." + print( + f'rid={rid} time={curr_t} time_delta={curr_t - request_start_time} message="request end" output="{str(output_partial)}"' + ) + + if pbar: + pbar.update(1) + return output + + +async def async_request_truss( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_request_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_sglang_generate( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + ("text" if isinstance(prompt, str) else "input_ids"): prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": request_func_input.output_len, + "ignore_eos": not args.disable_ignore_eos, + }, + "stream": not args.disable_stream, + "lora_path": request_func_input.lora_name, + "return_logprob": args.return_logprob, + "return_routed_experts": args.return_routed_experts, + "logprob_start_len": args.logprob_start_len, + **request_func_input.extra_request_body, + } + if args.top_logprobs_num > 0: + payload["top_logprobs_num"] = args.top_logprobs_num + if args.token_ids_logprob is not None: + payload["token_ids_logprob"] = args.token_ids_logprob + + # Add image data if available (list of image urls/base64) + if request_func_input.image_data: + payload["image_data"] = request_func_input.image_data + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + last_output_len = 0 + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if "text" in data and data["text"]: + timestamp = time.perf_counter() + generated_text = data["text"] + output_len = data["meta_info"]["completion_tokens"] + + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + num_new_tokens = output_len - last_output_len + if num_new_tokens == 0: + continue + chunk_gap = timestamp - most_recent_timestamp + adjust_itl = chunk_gap / num_new_tokens + output.itl.extend([adjust_itl] * num_new_tokens) + + most_recent_timestamp = timestamp + last_output_len = output_len + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + print(f"{output.error=}") + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_embeddings( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + async with _create_bench_client_session() as session: + payload = { + "input": request_func_input.prompt, + "model": request_func_input.model, + } + + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + payload.update(request_func_input.extra_request_body) + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + st = time.perf_counter() + output.start_time = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + await response.json() + output.latency = time.perf_counter() - st + output.success = True + output.output_len = 0 + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_gserver( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + raise NotImplementedError() + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with _create_bench_client_session() as session: + output = RequestFuncOutput() + try: + if api_url.endswith("/start_profile"): + num_steps = getattr(args, "profile_num_steps", None) + profile_by_stage = getattr(args, "profile_by_stage", None) + if profile_by_stage and num_steps is None: + num_steps = 5 + + output_dir = getattr(args, "profile_output_dir", None) + if output_dir is None: + output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp") + output_dir = Path(os.path.abspath(os.path.normpath(output_dir))) / str( + time.time() + ) + output_dir.mkdir(exist_ok=True, parents=True) + output_dir = str(output_dir) + + body = { + "activities": getattr(args, "profile_activities", []), + "num_steps": num_steps, + "profile_by_stage": profile_by_stage, + "profile_stages": getattr(args, "profile_stages", None), + "output_dir": output_dir, + "profile_prefix": getattr(args, "profile_prefix", None), + } + else: + # stop_profile doesn't need any parameters + body = {} + print(f"async_request_profile {api_url=} {body=}") + # Add optional profiling parameters if provided + if ( + hasattr(args, "profile_start_step") + and args.profile_start_step is not None + ): + body["start_step"] = str(args.profile_start_step) + if hasattr(args, "profile_steps") and args.profile_steps is not None: + body["num_steps"] = str(args.profile_steps) + async with session.post(url=api_url, json=body) as response: + if response.status == 200: + output.success = True + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def _build_profile_urls( + profile_prefill_url: Optional[List[str]], + profile_decode_url: Optional[List[str]], +) -> List[Tuple[str, str]]: + """Build profile URLs list from prefill/decode URL arguments. + + Returns: + List of (worker_type, url) tuples. e.g., [("Prefill-0", "http://..."), ("Decode-0", "http://...")] + """ + profile_urls = [] + if profile_prefill_url: + for idx, url in enumerate(profile_prefill_url): + profile_urls.append((f"Prefill-{idx}", url)) + if profile_decode_url: + for idx, url in enumerate(profile_decode_url): + profile_urls.append((f"Decode-{idx}", url)) + return profile_urls + + +async def _call_profile_pd(profile_urls: List[Tuple[str, str]], mode: str) -> None: + """Call profile endpoint (start/stop) on PD separated workers. + + Args: + profile_urls: List of (worker_type, url) tuples + mode: "start" or "stop" + """ + endpoint = "/start_profile" if mode == "start" else "/stop_profile" + action = "Starting" if mode == "start" else "Stopping" + action_past = "started" if mode == "start" else "stopped" + + print(f"{action} profiler...") + + for worker_type, url in profile_urls: + profile_output = await async_request_profile(api_url=url + endpoint) + if profile_output.success: + print(f"Profiler {action_past} for {worker_type} worker at {url}") + else: + print( + f"Failed to {mode} profiler for {worker_type} worker at {url}: {profile_output.error}" + ) + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_sglang_generate, + "sglang-native": async_request_sglang_generate, + "sglang-oai": async_request_openai_completions, + "sglang-oai-chat": async_request_openai_chat_completions, + "sglang-embedding": async_request_openai_embeddings, + "vllm": async_request_openai_completions, + "vllm-chat": async_request_openai_chat_completions, + "lmdeploy": async_request_openai_completions, + "lmdeploy-chat": async_request_openai_chat_completions, + "trt": async_request_trt_llm, + "gserver": async_request_gserver, + "truss": async_request_truss, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_input_text: int + total_input_vision: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p95_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p95_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p95_itl_ms: float + p99_itl_ms: float + max_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p90_e2e_latency_ms: float + p95_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + max_output_tokens_per_s: float = 0.0 + max_concurrent_requests: int = 0 + + +async def get_request( + input_requests: List[DatasetRow], + request_rate: float, + use_trace_timestamps: bool = False, + slowdown_factor: float = 1.0, +) -> AsyncGenerator[DatasetRow, None]: + if use_trace_timestamps: + print( + f"Using trace timestamps for request generation with slowdown factor {slowdown_factor}." + ) + # Sort requests by timestamp for correct replay + input_requests.sort(key=lambda r: r.timestamp) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0].timestamp if input_requests else 0 + + for request in input_requests: + trace_time_s = (request.timestamp - trace_start_time_ms) / 1000.0 + target_arrival_time = start_time + (trace_time_s * slowdown_factor) + + sleep_duration = target_arrival_time - time.perf_counter() + if sleep_duration > 0: + await asyncio.sleep(sleep_duration) + + yield request + else: + input_requests_iter = iter(input_requests) + for request in input_requests_iter: + yield request + + if request_rate == float("inf"): + # If the request rate is infinity, then we don't need to wait. + continue + + # Sample the request interval from the exponential distribution. + interval = np.random.exponential(1.0 / request_rate) + # The next request will be sent after the interval. + await asyncio.sleep(interval) + + +def calculate_metrics( + input_requests: Optional[List[DatasetRow]], + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, + accept_length: Optional[float] = None, + plot_throughput: bool = False, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + total_input_text = 0 + total_input_vision = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + retokenized_itls: List[float] = [] + + use_retokenized_itl = ( + accept_length is not None + and accept_length > 0 + and backend in ("sglang-oai", "sglang-oai-chat") + ) + + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_len + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode(outputs[i].generated_text, add_special_tokens=False) + ) + retokenized_output_lens.append(retokenized_output_len) + if input_requests is not None: + total_input += input_requests[i].prompt_len + total_input_text += input_requests[i].text_prompt_len + total_input_vision += input_requests[i].vision_prompt_len + if output_len > 1: + tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) + if use_retokenized_itl: + for k, itl in enumerate(outputs[i].itl): + num_tokens = len( + tokenizer.encode( + outputs[i].text_chunks[k], add_special_tokens=False + ) + ) + adjusted_itl = itl / num_tokens + retokenized_itls.extend([adjusted_itl] * num_tokens) + else: + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + + e2e_latencies.append(outputs[i].latency) + + completed += 1 + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + + max_output_tokens_per_s = 0.0 + max_concurrent_requests = 0 + + successful_outputs = [output for output in outputs if output.success] + if successful_outputs: + min_start_time = min(output.start_time for output in successful_outputs) + max_end_time = max( + output.start_time + output.latency for output in successful_outputs + ) + + duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1 + tokens_per_second = np.zeros(duration_seconds) + concurrent_requests_per_second = np.zeros(duration_seconds) + + for output in outputs: + if not output.success: + continue + + token_times = [output.start_time + output.ttft] + current_time = token_times[0] + for itl_value in output.itl: + current_time += itl_value + token_times.append(current_time) + + for token_time in token_times: + second_bucket = int(token_time - min_start_time) + if 0 <= second_bucket < duration_seconds: + tokens_per_second[second_bucket] += 1 + + request_start_second = int(output.start_time - min_start_time) + request_end_second = int( + (output.start_time + output.latency) - min_start_time + ) + + for second in range( + request_start_second, min(request_end_second + 1, duration_seconds) + ): + concurrent_requests_per_second[second] += 1 + + if len(tokens_per_second) > 0: + max_output_tokens_per_s = float(np.max(tokens_per_second)) + max_concurrent_requests = int(np.max(concurrent_requests_per_second)) + + if plot_throughput: + if TERM_PLOTLIB_AVAILABLE: + import termplotlib as tpl + + fig = tpl.figure() + fig.plot( + np.arange(len(tokens_per_second)), + tokens_per_second, + title="Output tokens per second", + xlabel="Time (s)", + ) + fig.plot( + np.arange(len(concurrent_requests_per_second)), + concurrent_requests_per_second, + title="Concurrent requests per second", + xlabel="Time (s)", + ) + fig.show() + else: + print("tip: install termplotlib and gnuplot to plot the metrics") + + itls = retokenized_itls if use_retokenized_itl else itls + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_input_text=total_input_text, + total_input_vision=total_input_vision, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p95_ttft_ms=np.percentile(ttfts or 0, 95) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p95_tpot_ms=np.percentile(tpots or 0, 95) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p95_itl_ms=np.percentile(itls or 0, 95) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + max_itl_ms=np.max(itls or 0) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p90_e2e_latency_ms=np.percentile(e2e_latencies, 90) * 1000, + p95_e2e_latency_ms=np.percentile(e2e_latencies, 95) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + max_output_tokens_per_s=max_output_tokens_per_s, + max_concurrent_requests=max_concurrent_requests, + ) + + return metrics, output_lens + + +MULTI_TURN_BACKENDS = {"sglang-oai-chat", "vllm-chat", "lmdeploy-chat"} + + +def wrap_multi_turn_request_func(request_func: Callable, backend: str) -> Callable: + assert ( + backend in MULTI_TURN_BACKENDS + ), f"Multi-turn only supports chat backends: {MULTI_TURN_BACKENDS}, got {backend}" + + async def f( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, + ) -> List[RequestFuncOutput]: + prompts: List[str] = request_func_input.prompt + prev_messages: List[Dict[str, str]] = [] + outputs = [] + + for round_index in range(len(prompts)): + prev_messages.append({"role": "user", "content": prompts[round_index]}) + + inner_input = replace( + copy.deepcopy(request_func_input), prompt=copy.deepcopy(prev_messages) + ) + output = await request_func( + inner_input, pbar=pbar if round_index == len(prompts) - 1 else None + ) + outputs.append(output) + + prev_messages.append( + {"role": "assistant", "content": output.generated_text} + ) + + return outputs + + return f + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[DatasetRow], + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_names: List[str], + lora_request_distribution: Optional[str], + lora_zipf_alpha: Optional[float], + extra_request_body: Dict[str, Any], + profile: bool, + pd_separated: bool = False, + flush_cache: bool = False, + warmup_requests: int = 1, + use_trace_timestamps: bool = False, + mooncake_slowdown_factor=1.0, + mooncake_num_rounds=1, + profile_prefill_url: Optional[List[str]] = None, + profile_decode_url: Optional[List[str]] = None, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Check for multi-turn: prompt is a list of strings (not OpenAI messages dicts) + # Multi-turn format: ["turn1", "turn2", ...] - list of strings + # OpenAI format: [{"role": "user", "content": "..."}, ...] - list of dicts + first_prompt = input_requests[0].prompt + is_multi_turn = ( + isinstance(first_prompt, list) + and len(first_prompt) > 0 + and isinstance(first_prompt[0], str) + ) + if is_multi_turn: + request_func = wrap_multi_turn_request_func(request_func, backend=backend) + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, pbar): + if semaphore is None: + return await request_func(request_func_input=request_func_input, pbar=pbar) + async with semaphore: + return await request_func(request_func_input=request_func_input, pbar=pbar) + + # Warmup + print(f"Starting warmup with {warmup_requests} sequences...") + + # Handle the data structure difference for the warmup request + if args.dataset_name == "mooncake": + # For mooncake, input_requests is a list of dicts. + # We need to build a temporary DatasetRow for the warmup phase. + warmup_record = input_requests[0] + + # Build prompt from hash_ids, just like in the async generator + hash_ids = warmup_record.get("hash_ids", []) + prompt_text = "" + for hash_id in hash_ids: + prompt_text += f"{hash_id}" + " ".join(["hi"] * 512) + prompt_text += "Can you tell me a detailed story in 1000 words?" + + output_len = warmup_record.get("output_length", 32) + prompt_len = len(tokenizer.encode(prompt_text)) + + # Create a temporary DatasetRow object for warmup + test_request = DatasetRow( + prompt=prompt_text, + prompt_len=prompt_len, + output_len=output_len, + image_data=None, # Mooncake doesn't have image data + ) + else: + # For all other datasets, input_requests is a list of DatasetRow objects + test_request = input_requests[0] + + if lora_names is not None and len(lora_names) != 0: + lora_name = lora_names[0] + else: + lora_name = None + + # Create the test input once + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=min(test_request.output_len, 32), + lora_name=lora_name, + image_data=test_request.image_data, + extra_request_body=extra_request_body, + ) + + # Run warmup requests + warmup_tasks = [] + for _ in range(warmup_requests): + warmup_tasks.append( + asyncio.create_task(request_func(request_func_input=test_input)) + ) + + warmup_outputs = await asyncio.gather(*warmup_tasks) + if is_multi_turn: + warmup_outputs = [x for output in warmup_outputs for x in output] + + # Check if at least one warmup request succeeded + if warmup_requests > 0 and not any(output.success for output in warmup_outputs): + raise ValueError( + "Warmup failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {warmup_outputs[0].error}" + ) + else: + print( + f"Warmup completed with {args.warmup_requests} sequences. Starting main benchmark run..." + ) + + # Flush cache + if ("sglang" in backend and _get_bool_env_var("SGLANG_IS_IN_CI")) or flush_cache: + requests.post(base_url + "/flush_cache", headers=get_auth_headers()) + + time.sleep(1.0) + + # Build profile URLs for PD separated mode (do this once at the beginning) + pd_profile_urls = [] + if profile and pd_separated: + pd_profile_urls = _build_profile_urls(profile_prefill_url, profile_decode_url) + if not pd_profile_urls: + print( + "Warning: PD separated mode requires --profile-prefill-url or --profile-decode-url" + ) + print("Skipping profiler start. Please specify worker URLs for profiling.") + + # Start profiler + if profile: + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "start") + else: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + # Run all requests + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + pbar_total = len(input_requests) + if ( + backend == "sglang" and args.dataset_name == "mooncake" + ): # Assuming mooncake is mainly for sglang or similar backends + print("Using time-based Mooncake request scheduler, ignoring --request-rate.") + request_generator = get_mooncake_request_over_time( + input_requests, tokenizer, mooncake_slowdown_factor, mooncake_num_rounds + ) + print( + f"Starting Mooncake trace replay. Sessions: {len(input_requests)}, Rounds per session: {mooncake_num_rounds}. Slowdown factor: {mooncake_slowdown_factor}" + ) + pbar_total *= args.mooncake_num_rounds + else: + request_generator = get_request(input_requests, request_rate) + + # Prepare LoRA request distribution parameters + if lora_request_distribution == "distinct": + lora_idx = 0 + elif lora_request_distribution == "skewed": + weights = np.array([lora_zipf_alpha**-i for i in range(len(lora_names))]) + lora_probs = weights / np.sum(weights) + else: + lora_idx = None + lora_probs = None + + pbar = None if disable_tqdm else tqdm(total=pbar_total) + async for request in request_generator: + if lora_names is not None and len(lora_names) != 0: + if lora_request_distribution == "uniform": + lora_name = random.choice(lora_names) + elif lora_request_distribution == "distinct": + lora_name = lora_names[lora_idx] + lora_idx = (lora_idx + 1) % len(lora_names) + else: + assert ( + lora_request_distribution == "skewed" + ), f"Unexpected lora_request_distribution: {lora_request_distribution}. Expected 'skewed'." + + lora_name = np.random.choice(lora_names, p=lora_probs) + else: + lora_name = None + + # Merge global extra_request_body with per-request extras + # Per-request parameters take precedence over global ones + merged_extra_body = {**extra_request_body, **request.extra_request_body} + + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name=lora_name, + image_data=request.image_data, + extra_request_body=merged_extra_body, + timestamp=request.timestamp, + routing_key=request.routing_key, + ) + + tasks.append( + asyncio.create_task( + limited_request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + if is_multi_turn: + outputs = [x for output in outputs for x in output] + + # Stop profiler (only if profile_steps was not provided, as it auto-stops) + if profile and not ( + hasattr(args, "profile_steps") and args.profile_steps is not None + ): + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "stop") + else: + if getattr(args, "profile_num_steps", None) is None: + print("Stopping profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/stop_profile" + ) + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + if "sglang" in backend: + server_info = requests.get( + base_url + "/server_info", headers=get_auth_headers() + ) + if server_info.status_code == 200: + server_info_json = server_info.json() + if "decode" in server_info_json: + server_info_json = server_info_json["decode"][0] + if ( + "internal_states" in server_info_json + and server_info_json["internal_states"] + ): + accept_length = server_info_json["internal_states"][0].get( + "avg_spec_accept_length", None + ) + else: + accept_length = None + else: + accept_length = None + else: + accept_length = None + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + input_requests=None if is_multi_turn else input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + accept_length=accept_length, + plot_throughput=args.plot_throughput, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print( + "{:<40} {:<10}".format( + "Traffic request rate:", "trace" if use_trace_timestamps else request_rate + ) + ) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total input text tokens:", metrics.total_input_text)) + if args.dataset_name in ["image", "mmmu"]: + print( + "{:<40} {:<10}".format( + "Total input vision tokens:", metrics.total_input_vision + ) + ) + is_embedding = backend == "sglang-embedding" + if not is_embedding: + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", + metrics.total_output_retokenized, + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + if not is_embedding: + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Peak output token throughput (tok/s):", + metrics.max_output_tokens_per_s, + ) + ) + print( + "{:<40} {:<10}".format( + "Peak concurrent requests:", metrics.max_concurrent_requests + ) + ) + if not is_embedding: + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + if accept_length: + print("{:<40} {:<10.2f}".format("Accept length:", accept_length)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print( + "{:<40} {:<10.2f}".format("P90 E2E Latency (ms):", metrics.p90_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format("P95 E2E Latency (ms):", metrics.p95_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format("P99 E2E Latency (ms):", metrics.p99_e2e_latency_ms) + ) + if not is_embedding: + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P95 TTFT (ms):", metrics.p95_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format( + s="Time per Output Token (excl. 1st token)", n=50, c="-" + ) + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P95 TPOT (ms):", metrics.p95_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-Token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P95 ITL (ms):", metrics.p95_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) + print("=" * 50) + + resp = requests.get(base_url + "/server_info", headers=get_auth_headers()) + server_info = resp.json() if resp.status_code == 200 else None + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "tag": getattr(args, "tag", None), + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": "trace" if use_trace_timestamps else request_rate, + "max_concurrency": max_concurrency, + "sharegpt_output_len": args.sharegpt_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Information + "server_info": server_info, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_input_text_tokens": metrics.total_input_text, + "total_input_vision_tokens": metrics.total_input_vision, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "total_throughput": metrics.total_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p90_e2e_latency_ms": metrics.p90_e2e_latency_ms, + "p95_e2e_latency_ms": metrics.p95_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p95_ttft_ms": metrics.p95_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p95_tpot_ms": metrics.p95_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p95_itl_ms": metrics.p95_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "accept_length": accept_length, + "max_output_tokens_per_s": metrics.max_output_tokens_per_s, + "max_concurrent_requests": metrics.max_concurrent_requests, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "image": + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_" + f"{args.random_output_len}_{args.image_count}imgs_" + f"{args.image_resolution}.jsonl" + ) + elif args.dataset_name.startswith("random"): + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + result_details = { + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + } + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + if args.output_details: + result_for_dump = result | result_details + else: + result_for_dump = result + file.write(json.dumps(result_for_dump) + "\n") + + return result | result_details + + +def check_chat_template(model_path): + try: + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + return "chat_template" in tokenizer.init_kwargs + except Exception as e: + print(f"Fail to load tokenizer config with error={e}") + return False + + +def set_global_args(args_: argparse.Namespace): + """Set the global args.""" + global args + args = args_ + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set default value for warmup_requests if not present + if not hasattr(args, "warmup_requests"): + args.warmup_requests = 1 + + if not hasattr(args, "output_details"): + args.output_details = False + + if not hasattr(args, "tokenize_prompt"): + args.tokenize_prompt = False + + if not hasattr(args, "plot_throughput"): + args.plot_throughput = False + + if not hasattr(args, "top_logprobs_num"): + args.top_logprobs_num = 0 + if not hasattr(args, "token_ids_logprob"): + args.token_ids_logprob = None + if not hasattr(args, "logprob_start_len"): + args.logprob_start_len = -1 + if not hasattr(args, "return_logprob"): + args.return_logprob = False + + if not hasattr(args, "use_trace_timestamps"): + args.use_trace_timestamps = False + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_num_rounds"): + args.mooncake_num_rounds = 1 + + if not hasattr(args, "served_model_name"): + args.served_model_name = None + + if getattr(args, "print_requests", False): + assert args.backend == "sglang-oai-chat" # only support this now + + print(f"benchmark_args={args}") + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + if args.tokenize_prompt: + assert ( + args.backend == "sglang" + ), "`--tokenize-prompt` only compatible with `--backend sglang` currently" + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "sglang-native": 30000, + "sglang-oai": 30000, + "lmdeploy": 23333, + "vllm": 8000, + "trt": 8000, + "gserver": 9988, + "truss": 8080, + }.get(args.backend, 30000) + + # Build base URL with proper IPv6 bracket wrapping (only when base_url is not provided) + if not args.base_url: + _na = NetworkAddress(args.host, args.port) + _host_base = _na.to_url() + else: + _na = None + _host_base = None + + model_url = ( + f"{args.base_url}/v1/models" if args.base_url else f"{_host_base}/v1/models" + ) + + if args.backend == "sglang-embedding": + api_url = ( + f"{args.base_url}/v1/embeddings" + if args.base_url + else f"http://{args.host}:{args.port}/v1/embeddings" + ) + elif args.backend in ["sglang", "sglang-native"]: + api_url = ( + f"{args.base_url}/generate" if args.base_url else f"{_host_base}/generate" + ) + elif args.backend in ["sglang-oai", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/completions" + if args.base_url + else f"{_host_base}/v1/completions" + ) + elif args.backend in ["sglang-oai-chat", "vllm-chat", "lmdeploy-chat"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"{_host_base}/v1/chat/completions" + ) + elif args.backend == "trt": + api_url = ( + f"{args.base_url}/v2/models/ensemble/generate_stream" + if args.base_url + else f"{_host_base}/v2/models/ensemble/generate_stream" + ) + if args.model is None: + print("Please provide a model using `--model` when using `trt` backend.") + sys.exit(1) + elif args.backend == "gserver": + api_url = args.base_url if args.base_url else _na.to_host_port_str() + args.model = args.model or "default" + elif args.backend == "truss": + api_url = ( + f"{args.base_url}/v1/models/model:predict" + if args.base_url + else f"{_host_base}/v1/models/model:predict" + ) + base_url = _host_base if args.base_url is None else args.base_url + + # Wait for server to be ready + if args.ready_check_timeout_sec > 0: + health_url = model_url if args.backend not in ("trt", "gserver") else base_url + if not wait_for_endpoint(health_url, args.ready_check_timeout_sec): + print(f"Server at {health_url} is not ready. Exiting.") + sys.exit(1) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url, headers=get_auth_headers()) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + if args.backend != "sglang-embedding" and not check_chat_template(args.model): + print( + "\nWARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.\n" + "Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.\n" + ) + + if ( + args.backend == "sglang-embedding" + and args.dataset_name in _EMBEDDING_UNSUPPORTED_DATASETS + ): + print(f"{args.dataset_name} dataset is unsupported for embeddings benchmark") + sys.exit(1) + + if args.dataset_name in ["image", "mmmu"]: + args.apply_chat_template = True + assert ( + not args.tokenize_prompt + ), "`--tokenize-prompt` not compatible with image dataset" + + if args.lora_request_distribution in ["distinct", "skewed"]: + assert ( + args.lora_name is not None and len(args.lora_name) > 1 + ), "More than 1 LoRA adapter must be specified via --lora-name to use 'distinct' or 'skewed' request distribution." + + assert ( + args.lora_zipf_alpha > 1 + ), f"Got invalid value for --lora-zipf-alpha of {args.lora_zipf_alpha}. It must be greater than 1." + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.served_model_name or args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + tokenizer = get_tokenizer(tokenizer_id) + input_requests = get_dataset(args, tokenizer, model_id) + + # compatible with SimpleNamespace + if not hasattr(args, "flush_cache"): + args.flush_cache = False + + # Prepare LoRA arguments + lora_request_distribution = ( + args.lora_request_distribution if args.lora_name is not None else None + ) + + lora_zipf_alpha = ( + args.lora_zipf_alpha + if args.lora_name is not None and args.lora_request_distribution == "skewed" + else None + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_names=args.lora_name, + lora_request_distribution=lora_request_distribution, + lora_zipf_alpha=lora_zipf_alpha, + extra_request_body=extra_request_body, + profile=args.profile, + pd_separated=args.pd_separated, + flush_cache=args.flush_cache, + warmup_requests=args.warmup_requests, + use_trace_timestamps=args.use_trace_timestamps, + mooncake_slowdown_factor=args.mooncake_slowdown_factor, + mooncake_num_rounds=args.mooncake_num_rounds, + profile_prefill_url=getattr(args, "profile_prefill_url", None), + profile_decode_url=getattr(args, "profile_decode_url", None), + ) + ) + + +class LoRAPathAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, []) + for lora_name in values: + getattr(namespace, self.dest).append(lora_name) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--ready-check-timeout-sec", + type=int, + default=60, + help="Maximum time in seconds to wait for the server to be ready before benchmarking. Set to 0 to skip. Default: 60.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "custom", + "openai", + "random", + "random-ids", + "generated-shared-prefix", + "mmmu", + "image", + "mooncake", + "longbench_v2", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--served-model-name", + type=str, + help="The name of the model as served by the serving service. If not set, this defaults to the value of --model.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random and image dataset.", + ) + # image dataset args + parser.add_argument( + "--image-count", + type=int, + default=1, + help="Number of images per request (only available with the image dataset)", + ) + parser.add_argument( + "--image-resolution", + type=str, + default="1080p", + help=( + "Resolution of images for image dataset. " + "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)." + ), + ) + parser.add_argument( + "--random-image-count", + action="store_true", + help="Enable Random Image Count", + ) + parser.add_argument( + "--image-format", + type=str, + default="jpeg", + help=("Format of images for image dataset. " "Supports jpeg and png."), + ) + parser.add_argument( + "--image-content", + type=str, + default="random", + help=("Content for images for image dataset. " "Supports random and blank."), + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--use-trace-timestamps", + action="store_true", + help="Use timestamps from the trace file for request scheduling. Only valid for 'mooncake' dataset.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--output-details", action="store_true", help="Output details of benchmarking." + ) + parser.add_argument( + "--print-requests", + action="store_true", + help="Print requests immediately during benchmarking. Useful to quickly realize issues.", + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument( + "--top-logprobs-num", + type=int, + default=0, + help="Number of top logprobs to return per token. Only used with --return-logprob.", + ) + parser.add_argument( + "--token-ids-logprob", + type=int, + nargs="+", + default=None, + help="Token IDs to probe logprobs for. E.g. --token-ids-logprob 1 2 10 100 1000. Only used with --return-logprob.", + ) + parser.add_argument( + "--logprob-start-len", + type=int, + default=-1, + help="Start position for returning input logprobs. -1 means no input logprobs, 0 means all. Only used with --return-logprob.", + ) + parser.add_argument( + "--return-routed-experts", + action="store_true", + help="Return routed experts.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--plot-throughput", + action="store_true", + help="Plot throughput and concurrent requests over time. Requires termplotlib and gnuplot.", + ) + # TODO unify all these + parser.add_argument( + "--profile-activities", + type=str, + nargs="+", + default=["CPU", "GPU"], + choices=["CPU", "GPU", "CUDA_PROFILER", "XPU"], + help="Profiler activities to capture: CPU, GPU, XPU, CUDA_PROFILER.", + ) + parser.add_argument( + "--profile-start-step", + type=int, + default=None, + help="Start profiling after this many forward steps. Useful for warmup.", + ) + parser.add_argument( + "--profile-steps", + type=int, + default=None, + help="Number of steps to profile. If specified, profiling stops automatically after this many steps.", + ) + parser.add_argument("--profile-num-steps", type=int, default=None) + parser.add_argument("--profile-by-stage", action="store_true", default=False) + parser.add_argument("--profile-stages", nargs="+", default=None) + parser.add_argument( + "--profile-output-dir", + type=str, + default=None, + help="Output directory for profile traces.", + ) + parser.add_argument( + "--profile-prefix", + type=str, + default=None, + help="Prefix for profile trace filenames.", + ) + parser.add_argument( + "--lora-name", + type=str, + nargs="*", + default=None, + action=LoRAPathAction, + help="The names of LoRA adapters. You can provide a list of names in the format {name} {name} {name}...", + ) + parser.add_argument( + "--lora-request-distribution", + type=str, + default="uniform", + choices=[ + "uniform", + "distinct", + "skewed", + ], + help="What distribution to sample the LoRA adapters specified in --lora-name. Borrowed from the Punica paper. " + "'distinct' distribution means selecting a new LoRA adapter for every request. " + "'skewed' distribution follows the Zipf distribution, where the number of requests " + "to model i specified in --lora-name is α times the number of requests for model i+1, " + "where α > 1.", + ) + parser.add_argument( + "--lora-zipf-alpha", + type=float, + default=1.5, + help="The parameter to use for the Zipf distribution when --lora-request-distribution='skewed'.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--pd-separated", + action="store_true", + help="Benchmark PD disaggregation server", + ) + + # Create a mutually exclusive group for profiling URLs + # In PD separated mode, prefill and decode workers must be profiled separately + profile_url_group = parser.add_mutually_exclusive_group() + profile_url_group.add_argument( + "--profile-prefill-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the prefill worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-prefill-url http://localhost:30000 http://localhost:30001. " + "NOTE: Cannot be used together with --profile-decode-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + profile_url_group.add_argument( + "--profile-decode-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the decode worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-decode-url http://localhost:30010 http://localhost:30011. " + "NOTE: Cannot be used together with --profile-prefill-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + parser.add_argument( + "--flush-cache", + action="store_true", + help="Flush the cache before running the benchmark", + ) + parser.add_argument( + "--warmup-requests", + type=int, + default=1, + help="Number of warmup requests to run before the benchmark", + ) + parser.add_argument( + "--tokenize-prompt", + action="store_true", + help="Use integer ids instead of string for inputs. Useful to control prompt lengths accurately", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + parser.add_argument( + "--gsp-range-ratio", + type=float, + # WARN: The default 1.0 is for backward compatibility, and is different from the default 0.0 for random dataset + default=1.0, + help="Range of sampled ratio of input/output length, used only for gsp dataset.", + ) + group.add_argument( + "--gsp-fast-prepare", + action="store_true", + help="Speedup preparing by removing statistics computation, which will make some output statistics inaccurate but suitable for pressure tests.", + ) + group.add_argument( + "--gsp-send-routing-key", + action="store_true", + help="Send routing key in requests via X-SMG-Routing-Key header. Requests with the same prefix share the same routing key.", + ) + group.add_argument( + "--gsp-num-turns", + type=int, + default=1, + help="Number of turns for multi-turn conversations. If > 1, each prompt becomes a list of questions sharing the same system prefix.", + ) + group.add_argument( + "--gsp-ordered", + action="store_true", + help="Keep requests in order without shuffling. By default, requests are shuffled randomly.", + ) + mooncake_group = parser.add_argument_group("mooncake dataset arguments") + mooncake_group.add_argument( + "--mooncake-slowdown-factor", + type=float, + default=1.0, + help="Slowdown factor for replaying the mooncake trace. " + "A value of 2.0 means the replay is twice as slow. " + "NOTE: --request-rate is IGNORED in mooncake mode.", + ) + mooncake_group.add_argument( + "--mooncake-num-rounds", + type=int, + default=1, + help="Number of conversation rounds for each session in the mooncake dataset. " + "A value > 1 will enable true multi-turn session benchmarking.", + ) + mooncake_group.add_argument( + "--mooncake-workload", + type=str, + default="conversation", + choices=[ + "mooncake", + "conversation", + "synthetic", + "toolagent", + ], + help="Underlying workload for the mooncake dataset.", + ) + parser.add_argument( + "--tag", type=str, default=None, help="The tag to be dumped to output." + ) + parser.add_argument( + "--header", + type=str, + nargs="+", + default=None, + help="Custom HTTP headers in Key=Value format. Example: --header MyHeader=MY_VALUE MyAnotherHeader=myanothervalue", + ) + args = parser.parse_args() + run_benchmark(args) diff --git a/experiments/p800/dsv4_p800_sglang_profile/run_profile.sh b/experiments/p800/dsv4_p800_sglang_profile/run_profile.sh new file mode 100755 index 0000000..ea68bb3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/run_profile.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Run a single profile benchmark on P800 SGLang with PyTorch profiler. +# ISL=4096, OSL=1024, concurrency=16, TP=8, DP=1 +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/config.env" + +TIMESTAMP="$(date '+%Y%m%d-%H%M%S')" + +# Override for this run +PROFILE_OUTPUT_DIR="${SCRIPT_DIR}/profile_output/${TIMESTAMP}" +BENCH_OUTPUT_DIR="${SCRIPT_DIR}/bench_output/${TIMESTAMP}" +mkdir -p "${PROFILE_OUTPUT_DIR}" +mkdir -p "${BENCH_OUTPUT_DIR}" + +# --------------------------------------------------------------------------- +# Step 0: Kill any existing container +# --------------------------------------------------------------------------- +log "Stopping any existing container: ${CONTAINER_NAME}" +docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true +sleep 2 + +# --------------------------------------------------------------------------- +# Step 1: Start the server (TP=8, DP=1) +# --------------------------------------------------------------------------- +log "Starting server (TP=8, DP=1)..." +bash "${SCRIPT_DIR}/start_sglang_docker.sh" 8 1 +log "Server is healthy" + +# --------------------------------------------------------------------------- +# Step 2: Start XPU monitoring in background +# --------------------------------------------------------------------------- +XPU_CSV="${PROFILE_OUTPUT_DIR}/xpu_monitor.csv" +log "Starting XPU monitor -> ${XPU_CSV}" +( + echo "timestamp,index,memory.used [MiB],memory.total [MiB],utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | python3 -c " +import sys, csv, io +data = sys.stdin.read() +lines = data.strip().split('\n') +ts = __import__('datetime').datetime.now().strftime('%Y/%m/%d %H:%M:%S') +for line in lines: + if 'MiB' in line and '%' in line: + parts = line.split('|') + idx = parts[1].strip() if len(parts) > 1 else '' + mem = parts[3].strip() if len(parts) > 3 else '' + util = parts[4].strip() if len(parts) > 4 else '' + if '/' in mem: + u = mem.split('/')[0].strip().replace('MiB','').strip() + t = mem.split('/')[1].strip().replace('MiB','').strip() + p = util.replace('%','').strip() + print(f'{ts}, {idx}, {u} MiB, {t} MiB, {p} %') + " + sleep 1 + done +) >> "${XPU_CSV}" 2>/dev/null & +XPU_MONITOR_PID=$! +log "XPU monitor PID=${XPU_MONITOR_PID}" + +# --------------------------------------------------------------------------- +# Step 3: Run bench_serving with --profile +# --------------------------------------------------------------------------- +log "Starting bench_serving with profiler..." +log " ISL=4096, OSL=1024, concurrency=16, num_prompts=80" +log " Profile steps=30, output=${PROFILE_OUTPUT_DIR}" + +BENCH_OUTPUT_FILE="/workspace/bench_output/results_${TIMESTAMP}.jsonl" +PROFILE_DIR_CONTAINER="/workspace/profile_output" + +set +e +docker exec "${CONTAINER_NAME}" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "${CONTAINER_PYTHON}" -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "${SGLANG_PORT}" \ + --model "${MODEL_PATH}" \ + --dataset-name random \ + --dataset-path "${CONTAINER_DATASET_PATH}" \ + --random-input-len 4096 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --num-prompts 80 \ + --max-concurrency 16 \ + --request-rate 10000 \ + --output-file "${BENCH_OUTPUT_FILE}" \ + --output-details \ + --profile \ + --profile-steps 30 \ + --profile-output-dir "${PROFILE_DIR_CONTAINER}" \ + --profile-prefix "p800_4k1k_c16" \ + 2>&1 | tee "${PROFILE_OUTPUT_DIR}/bench_serving.log" +BENCH_EXIT_CODE=$? +set -e + +log "bench_serving exit code: ${BENCH_EXIT_CODE}" + +# --------------------------------------------------------------------------- +# Step 4: Stop XPU monitor +# --------------------------------------------------------------------------- +kill "${XPU_MONITOR_PID}" 2>/dev/null || true +wait "${XPU_MONITOR_PID}" 2>/dev/null || true +log "XPU monitor stopped" + +# --------------------------------------------------------------------------- +# Step 5: Copy profile output files from container +# --------------------------------------------------------------------------- +log "Profile output files:" +docker exec "${CONTAINER_NAME}" find /tmp -name "*.json" -path "*profile*" 2>/dev/null \ + | while read -r f; do + log " Found in container: ${f}" + docker cp "${CONTAINER_NAME}:${f}" "${PROFILE_OUTPUT_DIR}/" 2>/dev/null || true + done + +docker exec "${CONTAINER_NAME}" ls -la "/workspace/profile_output/" 2>/dev/null +docker exec "${CONTAINER_NAME}" ls -la "/workspace/bench_output/" 2>/dev/null + +# Copy bench output +if docker exec "${CONTAINER_NAME}" test -f "${BENCH_OUTPUT_FILE}" 2>/dev/null; then + docker cp "${CONTAINER_NAME}:${BENCH_OUTPUT_FILE}" "${BENCH_OUTPUT_DIR}/" 2>/dev/null + log "Bench output copied" +fi + +# --------------------------------------------------------------------------- +# Step 6: List collected files +# --------------------------------------------------------------------------- +echo "" +echo "============================================" +echo "Collected files:" +echo "============================================" +find "${PROFILE_OUTPUT_DIR}" -type f 2>/dev/null || echo "(no profile files)" +find "${BENCH_OUTPUT_DIR}" -type f 2>/dev/null || echo "(no bench files)" + +# --------------------------------------------------------------------------- +# Step 7: Stop server +# --------------------------------------------------------------------------- +log "Stopping server container..." +docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true +log "Done! Profile output: ${PROFILE_OUTPUT_DIR}" \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_profile/start_sglang_docker.sh b/experiments/p800/dsv4_p800_sglang_profile/start_sglang_docker.sh new file mode 100755 index 0000000..7d86f65 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_profile/start_sglang_docker.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Start P800 SGLang INT8 server in Docker for profiling. +set -Eeuo pipefail + +TP="${1:-8}" +DP="${2:-1}" +if [[ -z "$TP" || -z "$DP" ]]; then + echo "Usage: $0 " + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/config.env" + +PORT="${SGLANG_PORT:-30015}" +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" +SERVER_LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" + +mkdir -p "${PROFILE_OUTPUT_DIR}" +mkdir -p "${BENCH_OUTPUT_DIR}" + +log "starting P800 SGLang INT8 server (tp=${TP}, dp=${DP})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "container: ${CONTAINER_NAME}" + +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables +env_args=( + -e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}" + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 + -e XPU_ENABLE_PROFILER_TRACING=1 +) + +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH}" + +if [[ "$DP" -gt 1 ]]; then + launch_args="${launch_args} --dp-size ${DP}" +fi + +# Write the server entrypoint script to a file (avoids shell quoting issues) +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +ENTRYPOINT_FILE="${RUNTIME_BASE}/entrypoint_${TIMESTAMP}.sh" +cat > "$ENTRYPOINT_FILE" << 'ENTRYEOF' +#!/usr/bin/env bash +set -Eeuo pipefail +cd /workspace +find /root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q 2>/dev/null +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" LAUNCH_ARGS_PLACEHOLDER +ENTRYEOF + +# Insert the launch args +sed -i "s|LAUNCH_ARGS_PLACEHOLDER|${launch_args}|g" "$ENTRYPOINT_FILE" +chmod +x "$ENTRYPOINT_FILE" + +# Patch mounts +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" + -v "${PATCH_ROOT}/dummy_sharegpt.json:/workspace/dummy_sharegpt.json:ro" + -v "${PATCH_ROOT}/sitecustomize_xpu.py:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sitecustomize.py:ro" + -v "${PATCH_ROOT}/hf_transformers_utils.py.patched:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/utils/hf_transformers_utils.py:ro" + -v "${PATCH_ROOT}/fp8_utils.py.patched:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/layers/quantization/fp8_utils.py:ro" + -v "${PATCH_ROOT}/config_backup_small_w8a8_int8.json:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/configs/config_backup_small.json:ro" + -v "${PATCH_ROOT}/parallel_state.py:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/distributed/parallel_state.py:ro" + -v "${PATCH_ROOT}/vocab_parallel_embedding.py:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/srt/layers/vocab_parallel_embedding.py:ro" + -v "${ENTRYPOINT_FILE}:/workspace/entrypoint.sh:ro" +) +BENCH_SERVING_PATCH="${SCRIPT_DIR}/patches/bench_serving.py" +if [[ -f "${BENCH_SERVING_PATCH}" ]]; then + patch_mounts+=(-v "${BENCH_SERVING_PATCH}:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/bench_serving.py:ro") +fi +if [[ -f "${DATASET_PATH}" ]]; then + patch_mounts+=(-v "${DATASET_PATH}:${CONTAINER_DATASET_PATH}:ro") +fi +patch_mounts+=(-v "${PROFILE_OUTPUT_DIR}:/workspace/profile_output:rw") +patch_mounts+=(-v "${BENCH_OUTPUT_DIR}:/workspace/bench_output:rw") + +echo "=== Starting P800 SGLang INT8 server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: ${DOCKER_IMAGE}" +echo "Container name: ${CONTAINER_NAME}" +echo "Host port: ${PORT}" + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash /workspace/entrypoint.sh \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +healthy=0 +for ((i = 1; i <= 600; i++)); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + healthy=1 + break + fi + if [[ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}" 2>/dev/null)" != "true" ]]; then + log "ERROR: container exited during startup:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 + fi + sleep 1 +done +if [[ "$healthy" == "1" ]]; then + log "container ${CONTAINER_NAME} is healthy" + exit 0 +else + log "ERROR: container failed health check:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 +fi \ No newline at end of file diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/README.md b/experiments/p800/dsv4_p800_sglang_tp_dp_official/README.md new file mode 100644 index 0000000..c039c26 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/README.md @@ -0,0 +1,83 @@ +# DSV4 P800 SGLang TP×DP Matrix（官方推荐参数版） + +Kunlun P800 XPU + SGLang-XPU + `DeepSeek-V4-Flash-INT8` TP×DP matrix benchmark。 +与 `dsv4_p800_sglang_tp_dp_matrix` 的区别仅在服务端启动参数：本目录使用 +**厂商官方推荐参数**（提取自镜像内 `/workspace/ds_v4/w8a8/run_server.sh`）： + +- `--ep-size`（仅 DP=1 时传入）：官方生产配置为 TP8/EP8。注意：**该镜像的 + EP 不能跨 DP 副本**——DP>1 时传 `--ep-size 8` 会在 + `data_parallel_controller.py` 触发 ZeroDivisionError（2026-07-17 实测）。 + 因此 TP2/DP4 依然不可行（EP=2 时每卡 132 GiB > 96 GiB），本矩阵只跑 + TP4/DP2 与 TP8/DP1 两种配置。 +- EAGLE 投机解码：`--speculative-algorithm EAGLE --speculative-num-steps 3 + --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` + （官方实测 accept length ≈ 2.5）。 +- `--chunked-prefill-size 8192 --max-prefill-tokens 16384`：长 prefill 切片调度。 +- `--max-running-requests 64`。 + +目的：量化官方参数相对此前实验参数的提升（TPOT / 长输入 TTFT / TP2 可行性）。 +其余（shape 矩阵、自适应搜索规则、结果结构）与 `dsv4_p800_sglang_tp_dp_matrix` +完全一致，结果可直接对比。 + +Differences from the H20 version: + +- Server runs in the P800 vendor image with the proven INT8 launch args + (NSA attention backend, `w8a8_int8` quantization); + the benchmark client runs inside the same container via `docker exec`. +- ISL is capped at 131072 because P800 INT8 sustains at most ~131k input + tokens; concurrency ranges match the H20 matrix (up to 128 at ISL=1024). + Scenarios beyond server capacity are recorded as failed/skipped and the + run moves on. +- XPU memory sampling uses `xpu-smi` instead of `nvidia-smi`. + +`--dp-size` support on this image was probed empirically (2026-07-16, +`results/smoke_tpdp`): TP=4/DP=2 starts and serves correctly (replicas land +on disjoint XPU groups), while TP=2/DP=4 fails during model loading with a +weight-loading OOM. Root cause: expert weights are sharded only across +the TP group (this matrix passes no `--ep-size`, so experts fall back to +TP sharding; the image also asserts `ep_size <= tp_size`, so EP can never +exceed TP anyway) and DP replicas do not share expert shards. The INT8 +checkpoint holds ~264 GiB of routed-expert weights out of ~274 GiB total, +so each rank carries roughly `274/TP` GiB: +TP=8/DP=1 ≈ 34 GiB, TP=4/DP=2 ≈ 68 GiB, TP=2/DP=4 ≈ 137 GiB — more than +the 96 GiB an XPU has, hence the OOM. TP=4 is therefore the +minimum viable TP for this model on P800. Configs that cannot start are +recorded as `SKIPPED_SERVICE_START_FAILED` and the run continues with the +next config. + +## Quick Start + +```bash +# Full matrix (server per config -> warmup -> scenarios -> parse -> compare) +bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh + +# Dry run: print server args and scenario plan only +DRY_RUN=1 bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_bench.sh + +# Adaptive concurrency search (find saturation point per shape) +bash experiments/p800/dsv4_p800_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +``` + +Results land in `results//` (fixed matrix) or +`adaptive_results//` (adaptive search). + +## Files + +| File | Purpose | +|---|---| +| `config.env` | Experiment-level configuration (model, port, TP×DP configs, server settings) | +| `matrix.json` | ISL/OSL matrix with Y/P/N marks and per-ISL concurrency ranges | +| `generate_scenarios.py` | Expands matrix.json into a scenario TSV | +| `start_sglang_docker.sh` | Start the P800 SGLang Docker container for a TP×DP config | +| `start_sglang_dp.sh` | Alias for `start_sglang_docker.sh` (H20 naming parity) | +| `run_bench.sh` | Orchestrator: server per config → warmup → scenarios → parse → compare | +| `compare.py` | Cross-config comparison report (`comparison.md`) | +| `adaptive_config.env` | Adaptive concurrency search settings | +| `run_adaptive_concurrency.sh` | Adaptive saturation search (shared `adaptive_bench_lib.sh`) | +| `run_adaptive_concurrency_add16.sh` | Same search starting at concurrency 16, step +16 | + +## Platform + +This experiment targets `platforms/kunlun_p800.env`. The container uses the +dedicated name `sglang-dsv4-flash-tpdp` so it never interferes with the +`sglang-dsv4-flash` container used by the other P800 experiments. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_config.env new file mode 100644 index 0000000..5f0140b --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_config.env @@ -0,0 +1,58 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +# No server-side max-running-requests cap is set; aligned with the H20 +# adaptive search upper bound. Points beyond server capacity are recorded +# as failed and the search moves on. +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# TTFT SLO early-stop settings. +# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# shape and move on to the next scenario. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# NOTE: the P800 image's sglang.bench_serving does not support +# --warmup-requests, so per-point warmup requests are not sent; the value is +# still validated by adaptive_bench_lib.sh and kept for parity with H20. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/run_manifest.json new file mode 100644 index 0000000..32e00ed --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "adaptive_20260717-072013", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072013/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.md b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/run_manifest.json new file mode 100644 index 0000000..f5ee62c --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "adaptive_20260717-072159", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-072159/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/run_manifest.json new file mode 100644 index 0000000..f2b4f4d --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "adaptive_20260717-073006", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-073006/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_points.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_shapes.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.md b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.md new file mode 100644 index 0000000..1ba5678 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/adaptive_summary.md @@ -0,0 +1,6 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/run_manifest.json new file mode 100644 index 0000000..d1d902b --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "adaptive_20260717-075355", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075355/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_points.jsonl new file mode 100644 index 0000000..096efb3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_points.jsonl @@ -0,0 +1,99 @@ +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":38.850977536989376,"request_tps":2.059150247219219,"input_tps":2108.5698531524804,"output_tps":263.57123164406005,"total_tps":2372.1410847965403,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":375.5967851029709,"ttft_p95_ms":5191.183777351398,"ttft_p99_ms":5494.083781067748,"tpot_p50_ms":48.15443845256782,"tpot_p95_ms":63.54774250823081,"tpot_p99_ms":70.50687441087186,"e2e_p50_ms":6587.240772438236,"e2e_p95_ms":12055.057296273297,"e2e_p99_ms":13356.3372077886,"itl_p50_ms":27.370366791728884,"itl_p95_ms":142.97846250701696,"itl_p99_ms":289.74504962873954,"validation_errors":[],"timestamp":"2026-07-17T07:59:52+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":24.875072153052315,"request_tps":1.6080355366965948,"input_tps":1646.628389577313,"output_tps":205.82854869716414,"total_tps":1852.4569382744771,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":342.23642048891634,"ttft_p95_ms":1023.3174284454436,"ttft_p99_ms":1165.7853273814542,"tpot_p50_ms":34.067742362449785,"tpot_p95_ms":40.35725147201216,"tpot_p99_ms":47.791466971980334,"e2e_p50_ms":4812.845256412402,"e2e_p95_ms":5945.917596516664,"e2e_p99_ms":6398.701484173071,"itl_p50_ms":20.754446974024177,"itl_p95_ms":90.81794312223788,"itl_p99_ms":180.40624327026327,"validation_errors":[],"timestamp":"2026-07-17T08:00:37+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":56.70565206790343,"request_tps":1.4107941110385653,"input_tps":1444.653169703491,"output_tps":361.1632924258727,"total_tps":1805.8164621293636,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":353.3130220603198,"ttft_p95_ms":1907.358543772716,"ttft_p99_ms":1910.2728008292615,"tpot_p50_ms":37.91363587621234,"tpot_p95_ms":46.84887420564122,"tpot_p99_ms":51.577967088916004,"e2e_p50_ms":10329.861202044412,"e2e_p95_ms":12712.218005850445,"e2e_p99_ms":14176.053717974562,"itl_p50_ms":24.598574731498957,"itl_p95_ms":95.18855294833578,"itl_p99_ms":178.34910145029426,"validation_errors":[],"timestamp":"2026-07-17T08:01:55+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":658.0221962139476,"request_tps":0.24315289198539136,"input_tps":248.98856139304075,"output_tps":62.24714034826019,"total_tps":311.23570174130094,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":2375.0393349910155,"ttft_p95_ms":6374.220354005225,"ttft_p99_ms":7099.810243677347,"tpot_p50_ms":358.5047604315275,"tpot_p95_ms":1659.2170038060551,"tpot_p99_ms":2032.845752709198,"e2e_p50_ms":96956.49567013606,"e2e_p95_ms":427836.4118320051,"e2e_p99_ms":523916.80456324946,"itl_p50_ms":284.34373397612944,"itl_p95_ms":1197.6450297981464,"itl_p99_ms":6089.286311843905,"validation_errors":[],"timestamp":"2026-07-17T08:13:13+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":-82.764821,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c32_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c32_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":95.85385624691844,"request_tps":0.8346038764879831,"input_tps":854.6343695236947,"output_tps":427.31718476184733,"total_tps":1281.951554285542,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":339.90226255264133,"ttft_p95_ms":2043.7121676397512,"ttft_p99_ms":2046.3871875125915,"tpot_p50_ms":34.03399106449541,"tpot_p95_ms":39.92383729727071,"tpot_p99_ms":44.13223402607008,"e2e_p50_ms":17862.977570970543,"e2e_p95_ms":21575.54897943046,"e2e_p99_ms":22888.442552858032,"itl_p50_ms":23.5583473307391,"itl_p95_ms":82.56516652181745,"itl_p99_ms":142.30121159460396,"validation_errors":[],"timestamp":"2026-07-17T08:15:10+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":1333.6260307179764,"request_tps":0.11997366301695667,"input_tps":122.85303092936363,"output_tps":61.42651546468181,"total_tps":184.27954639404544,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":2249.118781532161,"ttft_p95_ms":6836.219013039954,"ttft_p99_ms":7274.92825235473,"tpot_p50_ms":402.80989693945236,"tpot_p95_ms":1466.5643998654552,"tpot_p99_ms":2007.9641384794784,"e2e_p50_ms":207550.27348955628,"e2e_p95_ms":756035.7196734749,"e2e_p99_ms":1030145.6660308479,"itl_p50_ms":291.4304529937605,"itl_p95_ms":1255.9095733449797,"itl_p99_ms":6540.265160310082,"validation_errors":[],"timestamp":"2026-07-17T08:37:44+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":-85.625077,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c32_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c32_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":177.24370902986266,"request_tps":0.4513559349320619,"input_tps":462.1884773704314,"output_tps":462.1884773704314,"total_tps":924.3769547408627,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":341.96062106639147,"ttft_p95_ms":2057.607007981278,"ttft_p99_ms":2060.7414380577393,"tpot_p50_ms":32.20496962997672,"tpot_p95_ms":35.99809466682298,"tpot_p99_ms":38.50099444516569,"e2e_p50_ms":33410.80566099845,"e2e_p95_ms":37786.415580648456,"e2e_p99_ms":39677.768702611786,"itl_p50_ms":23.55353666159014,"itl_p95_ms":73.51064588874578,"itl_p99_ms":113.19450628322859,"validation_errors":[],"timestamp":"2026-07-17T08:41:03+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o1024_a1.log"} +{"timestamp":"2026-07-17T09:45:05+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":2,"status":"FAILED","completed":0,"failed":160,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c32_i1024_o1024_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c32_i1024_o1024_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":344.53868672205135,"request_tps":0.23219453455610967,"input_tps":237.7672033854563,"output_tps":475.5344067709126,"total_tps":713.301610156369,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":338.4241299936548,"ttft_p95_ms":5090.250183374155,"ttft_p99_ms":5357.846618075855,"tpot_p50_ms":31.61162330943483,"tpot_p95_ms":35.93945629929083,"tpot_p99_ms":37.76207400394197,"e2e_p50_ms":65312.21951346379,"e2e_p95_ms":76375.50360694295,"e2e_p99_ms":78678.96439327157,"itl_p50_ms":24.16073833592236,"itl_p95_ms":60.57862797752023,"itl_p99_ms":92.1715474280064,"validation_errors":[],"timestamp":"2026-07-17T09:55:51+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":295.84478807100095,"request_tps":0.13520603239560958,"input_tps":138.4509771731042,"output_tps":276.9019543462084,"total_tps":415.35293151931256,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":334.3576556071639,"ttft_p95_ms":1104.6332303900272,"ttft_p99_ms":1106.091428117361,"tpot_p50_ms":27.26071533514073,"tpot_p95_ms":30.96148045943425,"tpot_p99_ms":35.92901739716561,"e2e_p50_ms":56127.32804194093,"e2e_p95_ms":63730.06825210758,"e2e_p99_ms":73867.1764739533,"itl_p50_ms":20.85435518529266,"itl_p95_ms":54.84594932446877,"itl_p99_ms":82.91841029228388,"validation_errors":[],"timestamp":"2026-07-17T10:01:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":667.1058445849922,"request_tps":0.11992100001727335,"input_tps":122.79910401768791,"output_tps":491.19641607075164,"total_tps":613.9955200884395,"mean_input_tokens":1024.0,"mean_output_tokens":4096.0,"ttft_p50_ms":336.9323564693332,"ttft_p95_ms":2047.331060108263,"ttft_p99_ms":2051.3214324694127,"tpot_p50_ms":31.041202398263806,"tpot_p95_ms":34.29176860706998,"tpot_p99_ms":35.74743827387419,"e2e_p50_ms":127529.22121004667,"e2e_p95_ms":140774.1286351811,"e2e_p99_ms":146697.47561701105,"itl_p50_ms":24.100730661302805,"itl_p95_ms":59.010645685096584,"itl_p99_ms":87.47522998601198,"validation_errors":[],"timestamp":"2026-07-17T10:12:36+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i1024_o4096_a1.log"} +{"timestamp":"2026-07-17T11:17:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":2,"status":"FAILED","completed":0,"failed":160,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c32_i1024_o4096_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c32_i1024_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":261.27023342088796,"request_tps":0.306196381243039,"input_tps":1254.1803775714877,"output_tps":39.19313679910899,"total_tps":1293.3735143705967,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":6600.870855036192,"ttft_p95_ms":16131.13452475518,"ttft_p99_ms":24763.767675568793,"tpot_p50_ms":62.88574983120169,"tpot_p95_ms":945.8888633721177,"tpot_p99_ms":1395.5274336414748,"e2e_p50_ms":19390.217499574646,"e2e_p95_ms":138983.4251518827,"e2e_p99_ms":183925.186233425,"itl_p50_ms":27.527865953743458,"itl_p95_ms":595.7014495157637,"itl_p99_ms":11130.163294831356,"validation_errors":[],"timestamp":"2026-07-17T11:26:11+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":41.66513013304211,"request_tps":0.9600354030402608,"input_tps":3932.3050108529083,"output_tps":122.88453158915338,"total_tps":4055.189542442062,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":1204.7325940802693,"ttft_p95_ms":4109.041449218057,"ttft_p99_ms":4129.553488588426,"tpot_p50_ms":47.861005664495096,"tpot_p95_ms":74.32073753466463,"tpot_p99_ms":82.44492399761101,"e2e_p50_ms":8030.7446154765785,"e2e_p95_ms":12034.619017411023,"e2e_p99_ms":12552.433790641371,"itl_p50_ms":20.8846980240196,"itl_p95_ms":238.81182104232755,"itl_p99_ms":494.2852203012444,"validation_errors":[],"timestamp":"2026-07-17T11:27:14+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":19.41722331987694,"request_tps":0.2575033472927935,"input_tps":1054.7337105112822,"output_tps":32.96042845347757,"total_tps":1087.6941389647598,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":966.5790069848299,"ttft_p95_ms":997.8272448293865,"ttft_p99_ms":999.6989225782454,"tpot_p50_ms":23.180886936249344,"tpot_p95_ms":23.533233206381833,"tpot_p99_ms":23.588726540411436,"e2e_p50_ms":3920.0613570865244,"e2e_p95_ms":3944.4132342003286,"e2e_p99_ms":3944.467984456569,"itl_p50_ms":17.555401660501957,"itl_p95_ms":55.32496337157985,"itl_p99_ms":89.2760981572791,"validation_errors":[],"timestamp":"2026-07-17T11:27:55+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":387.94723369693384,"request_tps":0.20621361115954334,"input_tps":844.6509513094895,"output_tps":52.790684456843096,"total_tps":897.4416357663325,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":6097.636603051797,"ttft_p95_ms":21615.48103574896,"ttft_p99_ms":25336.934260143895,"tpot_p50_ms":44.969149533312255,"tpot_p95_ms":795.5569138228161,"tpot_p99_ms":868.5057627791168,"e2e_p50_ms":21400.709100067616,"e2e_p95_ms":211513.88580417258,"e2e_p99_ms":231401.03439270976,"itl_p50_ms":22.6946286081026,"itl_p95_ms":484.6375990891829,"itl_p99_ms":1306.0514004388826,"validation_errors":[],"timestamp":"2026-07-17T11:34:44+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":264.0962910801172,"request_tps":0.15145990818881078,"input_tps":620.3797839413689,"output_tps":38.77373649633556,"total_tps":659.1535204377045,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":1101.4024585019797,"ttft_p95_ms":4252.568581083324,"ttft_p99_ms":4335.392178189941,"tpot_p50_ms":41.4554171076994,"tpot_p95_ms":786.9617660628064,"tpot_p99_ms":825.2109340483634,"e2e_p50_ms":12043.69958746247,"e2e_p95_ms":204796.45780056014,"e2e_p99_ms":212503.5924624931,"itl_p50_ms":26.141325943171978,"itl_p95_ms":631.9959620013833,"itl_p99_ms":4480.901220631798,"validation_errors":[],"timestamp":"2026-07-17T11:39:29+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":33.375389450928196,"request_tps":0.14981098594674064,"input_tps":613.6257984378497,"output_tps":38.3516124023656,"total_tps":651.9774108402153,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":963.3728060871363,"ttft_p95_ms":994.264001538977,"ttft_p99_ms":995.9737562667578,"tpot_p50_ms":22.617584356453783,"tpot_p95_ms":22.96360349556541,"tpot_p99_ms":22.972889828367855,"e2e_p50_ms":6730.85681698285,"e2e_p95_ms":6834.964006580412,"e2e_p99_ms":6839.216810911894,"itl_p50_ms":17.6061923460414,"itl_p95_ms":50.265292148105786,"itl_p99_ms":80.49186949152498,"validation_errors":[],"timestamp":"2026-07-17T11:40:23+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":454.13498112000525,"request_tps":0.1761590789652471,"input_tps":721.5475874416521,"output_tps":90.19344843020652,"total_tps":811.7410358718586,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":5733.563286950812,"ttft_p95_ms":24313.488055858757,"ttft_p99_ms":33261.2008603569,"tpot_p50_ms":53.52417704714882,"tpot_p95_ms":418.4638156344861,"tpot_p99_ms":530.4572520386691,"e2e_p50_ms":44193.42886551749,"e2e_p95_ms":236812.54154903578,"e2e_p99_ms":284587.04277315637,"itl_p50_ms":21.92429592832923,"itl_p95_ms":426.2598494882691,"itl_p99_ms":652.0680158934556,"validation_errors":[],"timestamp":"2026-07-17T11:48:19+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":136.30687691411003,"request_tps":0.2934554800577294,"input_tps":1201.9936463164597,"output_tps":150.24920578955746,"total_tps":1352.2428521060174,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":1033.9201454771683,"ttft_p95_ms":4280.638697289395,"ttft_p99_ms":8584.84190521063,"tpot_p50_ms":33.4671427923621,"tpot_p95_ms":119.14688856156411,"tpot_p99_ms":127.06735239781572,"e2e_p50_ms":18408.433723379858,"e2e_p95_ms":61867.70031838677,"e2e_p99_ms":65931.43178206403,"itl_p50_ms":20.948467388128243,"itl_p95_ms":83.37374543771148,"itl_p99_ms":484.0245791059024,"validation_errors":[],"timestamp":"2026-07-17T11:50:56+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":60.09003691887483,"request_tps":0.08320846942980417,"input_tps":340.8218907844779,"output_tps":42.602736348059736,"total_tps":383.4246271325376,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":966.1563010886312,"ttft_p95_ms":1111.864162189886,"ttft_p99_ms":1138.037522798404,"tpot_p50_ms":21.820243708187906,"tpot_p95_ms":23.19130687026668,"tpot_p99_ms":23.445581859633794,"e2e_p50_ms":12109.375125030056,"e2e_p95_ms":12804.184590885416,"e2e_p99_ms":12931.573206270114,"itl_p50_ms":17.621152723828953,"itl_p95_ms":50.39396916205683,"itl_p99_ms":75.30430948827416,"validation_errors":[],"timestamp":"2026-07-17T11:52:18+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":645.2940471710172,"request_tps":0.1239744893831296,"input_tps":507.79950851329886,"output_tps":126.94987712832472,"total_tps":634.7493856416236,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":11541.393700055778,"ttft_p95_ms":36092.5420160638,"ttft_p99_ms":49168.57430588218,"tpot_p50_ms":58.472642563064944,"tpot_p95_ms":260.43368201257414,"tpot_p99_ms":292.02914609261967,"e2e_p50_ms":74551.709978492,"e2e_p95_ms":274809.3917686026,"e2e_p99_ms":308494.7103482413,"itl_p50_ms":21.852880522298317,"itl_p95_ms":333.07899528881535,"itl_p99_ms":649.4433570187539,"validation_errors":[],"timestamp":"2026-07-17T12:03:24+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":227.5296539280098,"request_tps":0.1758012606684488,"input_tps":720.0819636979663,"output_tps":180.02049092449158,"total_tps":900.1024546224579,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":1024.492367519997,"ttft_p95_ms":5396.207828703335,"ttft_p99_ms":15520.503231449975,"tpot_p50_ms":30.577873479063285,"tpot_p95_ms":92.07069102009231,"tpot_p99_ms":95.81112480217787,"e2e_p50_ms":32826.72059140168,"e2e_p95_ms":95177.13772283169,"e2e_p99_ms":99011.51060435455,"itl_p50_ms":20.94529933917026,"itl_p95_ms":73.41329882619877,"itl_p99_ms":429.52465700606507,"validation_errors":[],"timestamp":"2026-07-17T12:07:33+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":118.10847037681378,"request_tps":0.04233396626040434,"input_tps":173.39992580261617,"output_tps":43.34998145065404,"total_tps":216.7499072532702,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":994.6296268608421,"ttft_p95_ms":1088.473111204803,"ttft_p99_ms":1103.9815166592598,"tpot_p50_ms":21.861273968794393,"tpot_p95_ms":23.873395091223138,"tpot_p99_ms":24.023858569507812,"e2e_p50_ms":23375.01435400918,"e2e_p95_ms":25411.198532022536,"e2e_p99_ms":25569.85408883542,"itl_p50_ms":17.626291373744607,"itl_p95_ms":50.64874002709985,"itl_p99_ms":78.6475691082886,"validation_errors":[],"timestamp":"2026-07-17T12:09:52+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":1024,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1285.706132386811,"request_tps":0.06222261680551089,"input_tps":254.8638384353726,"output_tps":127.4319192176863,"total_tps":382.2957576530589,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":7793.805056484416,"ttft_p95_ms":124175.85486546157,"ttft_p99_ms":138231.42833698072,"tpot_p50_ms":48.185812841243695,"tpot_p95_ms":391.94600915554025,"tpot_p99_ms":452.9251856430392,"e2e_p50_ms":184631.34694099426,"e2e_p95_ms":809854.9680798431,"e2e_p99_ms":934447.2333263742,"itl_p50_ms":21.130787984778483,"itl_p95_ms":326.7150693834991,"itl_p99_ms":647.2556053742301,"validation_errors":[],"timestamp":"2026-07-17T13:09:54+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":2,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o2048_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o2048_a2.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":372.80018895398825,"request_tps":0.10729608295594743,"input_tps":439.4847557875607,"output_tps":219.74237789378034,"total_tps":659.227133681341,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":1026.6309829894453,"ttft_p95_ms":4160.6437171809375,"ttft_p99_ms":5962.894028893204,"tpot_p50_ms":28.57513246998877,"tpot_p95_ms":63.51515862790016,"tpot_p99_ms":70.53252697219476,"e2e_p50_ms":59836.63068851456,"e2e_p95_ms":131157.52772355915,"e2e_p99_ms":145337.58330172626,"itl_p50_ms":21.046053695802886,"itl_p95_ms":63.74408647728443,"itl_p99_ms":319.56663957098544,"validation_errors":[],"timestamp":"2026-07-17T13:16:28+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":230.6859021577984,"request_tps":0.021674493123467073,"input_tps":88.77872383372113,"output_tps":44.389361916860565,"total_tps":133.1680857505817,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":993.2131401728839,"ttft_p95_ms":1027.8619995806366,"ttft_p99_ms":1032.4825071264058,"tpot_p50_ms":22.029729757701837,"tpot_p95_ms":22.886445710474636,"tpot_p99_ms":22.88831107231957,"e2e_p50_ms":46058.08126507327,"e2e_p95_ms":47874.10710458644,"e2e_p99_ms":47884.39341929741,"itl_p50_ms":17.651243678604562,"itl_p95_ms":50.13533138359586,"itl_p99_ms":74.61027207318693,"validation_errors":[],"timestamp":"2026-07-17T13:20:40+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i4096_o2048_a1.log"} +{"timestamp":"2026-07-17T14:26:47+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":2,"status":"FAILED","completed":0,"failed":80,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i4096_o4096_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i4096_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":398.18903579190373,"request_tps":0.20090960023773366,"input_tps":3291.7028902950283,"output_tps":25.71642883042991,"total_tps":3317.419319125458,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":34813.111786032096,"ttft_p95_ms":50994.72494738875,"ttft_p99_ms":51776.69036109233,"tpot_p50_ms":63.77387399543225,"tpot_p95_ms":1438.9730977091576,"tpot_p99_ms":1633.9532007594173,"e2e_p50_ms":50007.97615386546,"e2e_p95_ms":209375.69161899155,"e2e_p99_ms":232513.55465993745,"itl_p50_ms":20.347388306011755,"itl_p95_ms":855.6929830578144,"itl_p99_ms":2259.143225712469,"validation_errors":[],"timestamp":"2026-07-17T14:39:08+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":128.8539250199683,"request_tps":0.3104290381049802,"input_tps":5086.069360311995,"output_tps":39.73491687743746,"total_tps":5125.8042771894325,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":15077.470013988204,"ttft_p95_ms":19990.58625124162,"ttft_p99_ms":20313.58475169167,"tpot_p50_ms":62.88907362247165,"tpot_p95_ms":205.3378949474404,"tpot_p99_ms":216.97863508118536,"e2e_p50_ms":23563.007198856212,"e2e_p95_ms":36759.9375275895,"e2e_p99_ms":37623.6838325928,"itl_p50_ms":19.962330348789692,"itl_p95_ms":81.56439673621207,"itl_p99_ms":1562.8019995762347,"validation_errors":[],"timestamp":"2026-07-17T14:41:41+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":37.016470527043566,"request_tps":0.13507500657976265,"input_tps":2213.0689078028313,"output_tps":17.28960084220962,"total_tps":2230.3585086450407,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":4250.729497987777,"ttft_p95_ms":4521.552444016561,"ttft_p99_ms":4568.235608851537,"tpot_p50_ms":24.46130622823642,"tpot_p95_ms":26.38359353730939,"tpot_p99_ms":26.67880913308697,"e2e_p50_ms":7394.722510827705,"e2e_p95_ms":7693.616286572069,"e2e_p99_ms":7733.251958098263,"itl_p50_ms":17.69061495239536,"itl_p95_ms":54.494046683733664,"itl_p99_ms":132.34294909983817,"validation_errors":[],"timestamp":"2026-07-17T14:42:42+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":360.29490173910744,"request_tps":0.22204033311003848,"input_tps":3637.9088176748705,"output_tps":56.84232527616985,"total_tps":3694.7511429510405,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":36550.56051153224,"ttft_p95_ms":69809.02896203332,"ttft_p99_ms":73934.31270377243,"tpot_p50_ms":43.9642931389458,"tpot_p95_ms":431.683559238171,"tpot_p99_ms":465.1442075694874,"e2e_p50_ms":54476.55532252975,"e2e_p95_ms":153019.31679926347,"e2e_p99_ms":172444.38501281894,"itl_p50_ms":20.190449159902833,"itl_p95_ms":78.7381677655503,"itl_p99_ms":1430.1772827988445,"validation_errors":[],"timestamp":"2026-07-17T14:49:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":517.513580017956,"request_tps":0.07729265770883179,"input_tps":1266.3629039015,"output_tps":19.786920373460937,"total_tps":1286.149824274961,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":17595.85192555096,"ttft_p95_ms":23331.382640684023,"ttft_p99_ms":24793.476201938465,"tpot_p50_ms":45.038394963222686,"tpot_p95_ms":1517.6137506777384,"tpot_p99_ms":1585.4774185186525,"e2e_p50_ms":29910.490548005328,"e2e_p95_ms":403743.95331996493,"e2e_p99_ms":419686.6936565051,"itl_p50_ms":19.938393340756495,"itl_p95_ms":1137.352753255982,"itl_p99_ms":2330.0107532801726,"validation_errors":[],"timestamp":"2026-07-17T14:58:10+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":50.954981565941125,"request_tps":0.09812583277121732,"input_tps":1607.6936441236246,"output_tps":25.120213189431635,"total_tps":1632.8138573130561,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":4191.572895972058,"ttft_p95_ms":4522.999642696232,"ttft_p99_ms":4588.879505228251,"tpot_p50_ms":23.602170098171225,"tpot_p95_ms":23.939436206995858,"tpot_p99_ms":23.98344695575389,"e2e_p50_ms":10189.345370978117,"e2e_p95_ms":10554.832731094211,"e2e_p99_ms":10610.088822934777,"itl_p50_ms":17.734466353431344,"itl_p95_ms":54.632326044763126,"itl_p99_ms":88.35889969021073,"validation_errors":[],"timestamp":"2026-07-17T14:59:25+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":464.1876124527771,"request_tps":0.17234410797237418,"input_tps":2823.6858650193785,"output_tps":88.24018328185558,"total_tps":2911.926048301234,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":46347.98960306216,"ttft_p95_ms":89187.7718266449,"ttft_p99_ms":96087.79868721262,"tpot_p50_ms":42.43808294329318,"tpot_p95_ms":212.0776728536011,"tpot_p99_ms":296.1004276309112,"e2e_p50_ms":74489.89607742988,"e2e_p95_ms":159108.3795682178,"e2e_p99_ms":197463.44999968284,"itl_p50_ms":20.550117284680407,"itl_p95_ms":62.954848853405565,"itl_p99_ms":903.2773144915639,"validation_errors":[],"timestamp":"2026-07-17T15:07:34+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":217.00395834981464,"request_tps":0.18432843485518,"input_tps":3020.037076667269,"output_tps":94.37615864585216,"total_tps":3114.4132353131213,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":18161.11553600058,"ttft_p95_ms":29282.41261792136,"ttft_p99_ms":30154.66339093633,"tpot_p50_ms":36.77544146666847,"tpot_p95_ms":72.13047890937129,"tpot_p99_ms":97.35414838178698,"e2e_p50_ms":41115.67160242703,"e2e_p95_ms":59853.24986398918,"e2e_p99_ms":70428.34478157571,"itl_p50_ms":20.092259005953867,"itl_p95_ms":63.071597600355744,"itl_p99_ms":631.3162861492798,"validation_errors":[],"timestamp":"2026-07-17T15:11:35+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":79.87436655093916,"request_tps":0.06259830551283678,"input_tps":1025.6106375223178,"output_tps":32.05033242257243,"total_tps":1057.6609699448902,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":4214.033427182585,"ttft_p95_ms":4521.75477463752,"ttft_p99_ms":4565.007229968905,"tpot_p50_ms":22.794917475526933,"tpot_p95_ms":24.130802957519684,"tpot_p99_ms":24.17153189819007,"e2e_p50_ms":15862.236257176846,"e2e_p95_ms":16812.242742348462,"e2e_p99_ms":16908.58956122771,"itl_p50_ms":17.676378988350432,"itl_p95_ms":52.69915168173611,"itl_p99_ms":79.58481988636778,"validation_errors":[],"timestamp":"2026-07-17T15:13:20+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1033.0590840869118,"request_tps":0.07743990758351393,"input_tps":1268.7754458482923,"output_tps":79.29846536551827,"total_tps":1348.0739112138106,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":73881.86260208022,"ttft_p95_ms":107582.44929644279,"ttft_p99_ms":112520.41371415835,"tpot_p50_ms":33.94909385823438,"tpot_p95_ms":469.1952465789188,"tpot_p99_ms":612.5857616169103,"e2e_p50_ms":123860.53596646525,"e2e_p95_ms":539936.6624629241,"e2e_p99_ms":657626.2736182422,"itl_p50_ms":20.96131669046978,"itl_p95_ms":73.52309802081436,"itl_p99_ms":1499.6035116204707,"validation_errors":[],"timestamp":"2026-07-17T15:30:58+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":359.5240060000215,"request_tps":0.11125821734417815,"input_tps":1822.8546329670148,"output_tps":113.92841456043843,"total_tps":1936.7830475274534,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":21531.717553036287,"ttft_p95_ms":44257.4294752674,"ttft_p99_ms":50748.626555625815,"tpot_p50_ms":30.971451415880633,"tpot_p95_ms":94.47616205434879,"tpot_p99_ms":134.78354309001145,"e2e_p50_ms":67019.38616961706,"e2e_p95_ms":131295.3089486924,"e2e_p99_ms":151811.14625662332,"itl_p50_ms":20.126504357904196,"itl_p95_ms":57.439883006736636,"itl_p99_ms":104.5655637816524,"validation_errors":[],"timestamp":"2026-07-17T15:37:22+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":137.96030110912398,"request_tps":0.036242309996446696,"input_tps":593.7940069817827,"output_tps":37.11212543636142,"total_tps":630.9061324181441,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":4207.704800879583,"ttft_p95_ms":4496.435682615265,"ttft_p99_ms":4540.599406948313,"tpot_p50_ms":22.844187776264047,"tpot_p95_ms":23.61765927311947,"tpot_p99_ms":23.690314634647567,"e2e_p50_ms":27577.308895997703,"e2e_p95_ms":28657.301119016483,"e2e_p99_ms":28775.791278192773,"itl_p50_ms":17.715064731116097,"itl_p95_ms":50.49900479304294,"itl_p99_ms":79.0246335999108,"validation_errors":[],"timestamp":"2026-07-17T15:40:04+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":1024,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1500.8557320381515,"request_tps":0.05330292465309811,"input_tps":873.3151175163595,"output_tps":109.16438968954493,"total_tps":982.4795072059044,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":97297.29381052312,"ttft_p95_ms":182535.45828405768,"ttft_p99_ms":188002.87273567865,"tpot_p50_ms":47.49208928408209,"tpot_p95_ms":224.4996838377853,"tpot_p99_ms":264.081719150114,"e2e_p50_ms":224741.23554909602,"e2e_p95_ms":555453.9897803565,"e2e_p99_ms":611479.2148616326,"itl_p50_ms":20.20345100512107,"itl_p95_ms":58.55113454551122,"itl_p99_ms":730.0397504586726,"validation_errors":[],"timestamp":"2026-07-17T16:05:31+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":612.241017438937,"request_tps":0.06533374743058518,"input_tps":1070.4281179027075,"output_tps":133.80351473783844,"total_tps":1204.231632640546,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":54340.08844138589,"ttft_p95_ms":75981.57878243357,"ttft_p99_ms":106221.49156776955,"tpot_p50_ms":28.269403935767233,"tpot_p95_ms":61.76984594972993,"tpot_p99_ms":83.49832735894294,"e2e_p50_ms":117875.88013254572,"e2e_p95_ms":182506.47001382892,"e2e_p99_ms":187087.45992703128,"itl_p50_ms":20.03885196366658,"itl_p95_ms":56.91900298309823,"itl_p99_ms":86.03990694973618,"validation_errors":[],"timestamp":"2026-07-17T16:16:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":252.95843224599957,"request_tps":0.019766093407542745,"input_tps":323.8476743891803,"output_tps":40.48095929864754,"total_tps":364.32863368782785,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":4133.095419965684,"ttft_p95_ms":4547.52941490151,"ttft_p99_ms":4612.211770871654,"tpot_p50_ms":22.576647774809103,"tpot_p95_ms":23.856792781860523,"tpot_p99_ms":23.98585440125999,"e2e_p50_ms":50347.49341499992,"e2e_p95_ms":53382.38423937,"e2e_p99_ms":53711.25573025085,"itl_p50_ms":17.742878990247846,"itl_p95_ms":50.39254468865692,"itl_p99_ms":79.0636190213263,"validation_errors":[],"timestamp":"2026-07-17T16:20:45+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":975.513799786102,"request_tps":0.08200806592130358,"input_tps":5374.480608218551,"output_tps":10.497032437926858,"total_tps":5384.977640656479,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":145695.66617091186,"ttft_p95_ms":171858.59152760822,"ttft_p99_ms":187365.59422909748,"tpot_p50_ms":195.78209132282433,"tpot_p95_ms":1322.012301557422,"tpot_p99_ms":1651.3642695862175,"e2e_p50_ms":175110.83833896555,"e2e_p95_ms":298595.18617785064,"e2e_p99_ms":350518.7197246633,"itl_p50_ms":20.16425070663293,"itl_p95_ms":118.27680676942724,"itl_p99_ms":7031.445415453055,"validation_errors":[],"timestamp":"2026-07-17T16:37:43+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":65536,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c16_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c16_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":427.0155733949505,"request_tps":0.09367339856479577,"input_tps":6138.979848342456,"output_tps":11.990195016293859,"total_tps":6150.9700433587495,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":59328.99528858252,"ttft_p95_ms":81303.70528574567,"ttft_p99_ms":81365.60643166536,"tpot_p50_ms":184.73431326844914,"tpot_p95_ms":363.04904296975843,"tpot_p99_ms":496.3024316452385,"e2e_p50_ms":83123.13189904671,"e2e_p95_ms":106477.05610428238,"e2e_p99_ms":117460.91308631001,"itl_p50_ms":20.16357332468033,"itl_p95_ms":119.78968068335921,"itl_p99_ms":6638.133200273539,"validation_errors":[],"timestamp":"2026-07-17T16:45:29+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":65536,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c8_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c8_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":112.6869492130354,"request_tps":0.04437071049414487,"input_tps":2907.8788829442783,"output_tps":5.679450943250544,"total_tps":2913.5583338875285,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":19385.76221000403,"ttft_p95_ms":19571.64373351261,"ttft_p99_ms":19583.053931426257,"tpot_p50_ms":24.784649700278372,"tpot_p95_ms":26.589581887347727,"tpot_p99_ms":26.82453790914238,"e2e_p50_ms":22579.32162284851,"e2e_p95_ms":22932.51477042213,"e2e_p99_ms":22986.56907333061,"itl_p50_ms":17.727869329974055,"itl_p95_ms":58.14168299548328,"itl_p99_ms":115.8542741148264,"validation_errors":[],"timestamp":"2026-07-17T16:48:01+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":65536,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/raw_outputs/sglang_adaptive_c1_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp4_dp2/logs/sglang_c1_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":46.78171614790335,"request_tps":1.7100698004980182,"input_tps":1751.1114757099706,"output_tps":218.88893446374632,"total_tps":1970.0004101737168,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":644.706156454049,"ttft_p95_ms":3374.4750882382505,"ttft_p99_ms":3964.2573517840356,"tpot_p50_ms":58.15262539774238,"tpot_p95_ms":95.86080529939706,"tpot_p99_ms":110.60167424534774,"e2e_p50_ms":8008.67016997654,"e2e_p95_ms":15172.059130703565,"e2e_p99_ms":16239.937297182621,"itl_p50_ms":30.591263668611646,"itl_p95_ms":179.47053063350418,"itl_p99_ms":442.4362682620995,"validation_errors":[],"timestamp":"2026-07-17T16:54:50+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":53.41438919189386,"request_tps":2.995447526792678,"input_tps":3067.338267435702,"output_tps":383.41728342946277,"total_tps":3450.755550865165,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":548.8502155058086,"ttft_p95_ms":1378.2715240493417,"ttft_p99_ms":1382.0858210232109,"tpot_p50_ms":78.6861710545541,"tpot_p95_ms":109.2755255327875,"tpot_p99_ms":119.9757407159603,"e2e_p50_ms":10538.391953101382,"e2e_p95_ms":14656.311538314909,"e2e_p99_ms":15758.095573405735,"itl_p50_ms":43.27917343471199,"itl_p95_ms":240.85063968474665,"itl_p99_ms":419.8645495437088,"validation_errors":[],"timestamp":"2026-07-17T16:56:04+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":75.16522,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":109.16131979692727,"request_tps":2.19858096665075,"input_tps":2251.346909850368,"output_tps":281.418363731296,"total_tps":2532.7652735816637,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1118.607769603841,"ttft_p95_ms":5606.538144976365,"ttft_p99_ms":6509.578482515644,"tpot_p50_ms":157.07960657089743,"tpot_p95_ms":204.97110178263299,"tpot_p99_ms":230.9743633937075,"e2e_p50_ms":21525.960437604226,"e2e_p95_ms":28873.033440869767,"e2e_p99_ms":31532.613050478958,"itl_p50_ms":95.54555569775403,"itl_p95_ms":426.1157557484694,"itl_p99_ms":954.9172789487056,"validation_errors":[],"timestamp":"2026-07-17T16:58:14+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":-26.602588,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":63.953612270066515,"request_tps":1.2509066675103824,"input_tps":1280.9284275306316,"output_tps":320.2321068826579,"total_tps":1601.1605344132893,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":416.3627445232123,"ttft_p95_ms":2623.383231391199,"ttft_p99_ms":2624.797524828464,"tpot_p50_ms":44.68458674291112,"tpot_p95_ms":55.13308194744419,"tpot_p99_ms":59.3003548199183,"e2e_p50_ms":11877.51704256516,"e2e_p95_ms":15466.884967614897,"e2e_p99_ms":16354.233928658068,"itl_p50_ms":24.046485001842182,"itl_p95_ms":127.618253386269,"itl_p99_ms":232.81705901725218,"validation_errors":[],"timestamp":"2026-07-17T16:59:38+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":79.14570383285172,"request_tps":2.0215879353085917,"input_tps":2070.106045755998,"output_tps":517.5265114389995,"total_tps":2587.6325571949974,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":449.3000309448689,"ttft_p95_ms":1396.975072810892,"ttft_p99_ms":1399.5271435473114,"tpot_p50_ms":57.805774862622364,"tpot_p95_ms":74.3223027221621,"tpot_p99_ms":79.32276595428623,"e2e_p50_ms":15326.183204073459,"e2e_p95_ms":19415.795307327062,"e2e_p99_ms":20732.070125869937,"itl_p50_ms":38.515673484653234,"itl_p95_ms":169.70017133280635,"itl_p99_ms":313.5164436729004,"validation_errors":[],"timestamp":"2026-07-17T17:01:17+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":61.609814,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":171.05532362195663,"request_tps":1.4030548416628972,"input_tps":1436.7281578628067,"output_tps":359.1820394657017,"total_tps":1795.9101973285085,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":734.0757724596187,"ttft_p95_ms":6752.489144913852,"ttft_p99_ms":8109.708762406372,"tpot_p50_ms":126.20574137442469,"tpot_p95_ms":156.70472340895702,"tpot_p99_ms":166.14100317137456,"e2e_p50_ms":33469.33325752616,"e2e_p95_ms":41091.82116157608,"e2e_p99_ms":46133.33143891765,"itl_p50_ms":92.07488767181835,"itl_p95_ms":282.2985073241095,"itl_p99_ms":562.8461372091746,"validation_errors":[],"timestamp":"2026-07-17T17:04:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":-30.596398,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":109.54015564592555,"request_tps":0.7303257835290073,"input_tps":747.8536023337035,"output_tps":373.92680116685176,"total_tps":1121.7804035005554,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":333.35916395299137,"ttft_p95_ms":2634.9478750140406,"ttft_p99_ms":2636.4444935228676,"tpot_p50_ms":37.47759736780945,"tpot_p95_ms":46.18186015267117,"tpot_p99_ms":50.49830610891197,"e2e_p50_ms":19953.837350942194,"e2e_p95_ms":24739.065816998478,"e2e_p99_ms":26177.83488437065,"itl_p50_ms":23.97618629038334,"itl_p95_ms":99.07471656333655,"itl_p99_ms":177.9070011107251,"validation_errors":[],"timestamp":"2026-07-17T17:06:39+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":136.62853526999243,"request_tps":1.171058444590825,"input_tps":1199.1638472610048,"output_tps":599.5819236305024,"total_tps":1798.7457708915072,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":415.7929359935224,"ttft_p95_ms":1384.854385012295,"ttft_p99_ms":1387.649688264355,"tpot_p50_ms":49.88427390503914,"tpot_p95_ms":61.97650720699783,"tpot_p99_ms":69.7170729766492,"e2e_p50_ms":25977.437033550814,"e2e_p95_ms":32000.832956889637,"e2e_p99_ms":35987.0310997893,"itl_p50_ms":36.77442669868469,"itl_p95_ms":123.08880263008177,"itl_p99_ms":219.40896392334253,"validation_errors":[],"timestamp":"2026-07-17T17:09:17+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":60.347405,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":291.14597221207805,"request_tps":0.8243287660018802,"input_tps":844.1126563859253,"output_tps":422.05632819296267,"total_tps":1266.168984578888,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":570.156249916181,"ttft_p95_ms":6629.769576434046,"ttft_p99_ms":7916.567168761976,"tpot_p50_ms":108.80589346290218,"tpot_p95_ms":131.07439897336621,"tpot_p99_ms":143.22254358064518,"e2e_p50_ms":56996.836331440136,"e2e_p95_ms":69140.13169152895,"e2e_p99_ms":73940.08099251421,"itl_p50_ms":83.29143933951855,"itl_p95_ms":241.4768539019858,"itl_p99_ms":358.0241386468212,"validation_errors":[],"timestamp":"2026-07-17T17:14:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":-29.60823,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":195.7554185190238,"request_tps":0.40867323420845936,"input_tps":418.4813918294624,"output_tps":418.4813918294624,"total_tps":836.9627836589248,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":292.98997996374965,"ttft_p95_ms":2752.287955326028,"ttft_p99_ms":2753.923822857905,"tpot_p50_ms":34.98376812615167,"tpot_p95_ms":41.839741845797285,"tpot_p99_ms":47.64768215832649,"e2e_p50_ms":36506.47636002395,"e2e_p95_ms":43277.69151709508,"e2e_p99_ms":49066.59334763416,"itl_p50_ms":23.925696732476354,"itl_p95_ms":82.6315023121424,"itl_p99_ms":125.85630438989031,"validation_errors":[],"timestamp":"2026-07-17T17:18:05+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":237.58278739801608,"request_tps":0.673449460511448,"input_tps":689.6122475637228,"output_tps":689.6122475637228,"total_tps":1379.2244951274456,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":347.8804155020043,"ttft_p95_ms":1548.6168869072571,"ttft_p99_ms":1551.8960725027137,"tpot_p50_ms":43.83776438473028,"tpot_p95_ms":51.96910767838993,"tpot_p99_ms":54.632827966211806,"e2e_p50_ms":45368.03813499864,"e2e_p95_ms":53748.8272216753,"e2e_p99_ms":56190.46108613489,"itl_p50_ms":29.736904970680673,"itl_p95_ms":102.87870548199862,"itl_p99_ms":168.27299762594822,"validation_errors":[],"timestamp":"2026-07-17T17:22:24+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":64.789226,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":527.4060312591027,"request_tps":0.45505736714279893,"input_tps":465.9787439542261,"output_tps":465.9787439542261,"total_tps":931.9574879084522,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":527.3684379644692,"ttft_p95_ms":6623.470642627217,"ttft_p99_ms":7906.3602521736175,"tpot_p50_ms":99.32857738224696,"tpot_p95_ms":117.96208059451409,"tpot_p99_ms":122.59222529316003,"e2e_p50_ms":102544.36626192182,"e2e_p95_ms":121183.15940028988,"e2e_p99_ms":126035.59201440308,"itl_p50_ms":82.7716769805799,"itl_p95_ms":167.3658080321426,"itl_p99_ms":292.4512748885898,"validation_errors":[],"timestamp":"2026-07-17T17:31:33+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":-32.428876,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":370.6881436088588,"request_tps":0.21581483351788577,"input_tps":220.99438952231503,"output_tps":441.98877904463006,"total_tps":662.9831685669451,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":292.7126814611256,"ttft_p95_ms":2632.945351721719,"ttft_p99_ms":2634.5813917461783,"tpot_p50_ms":33.72283447506548,"tpot_p95_ms":39.80022201075255,"tpot_p99_ms":41.38097858816678,"e2e_p50_ms":69763.71911796741,"e2e_p95_ms":81798.67568440967,"e2e_p99_ms":86225.5317539442,"itl_p50_ms":24.285356979817152,"itl_p95_ms":77.97504193149504,"itl_p99_ms":114.18037889758135,"validation_errors":[],"timestamp":"2026-07-17T17:38:05+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":456.95510968612507,"request_tps":0.3501438032061866,"input_tps":358.5472544831351,"output_tps":717.0945089662702,"total_tps":1075.6417634494053,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":306.2125281430781,"ttft_p95_ms":1380.9562189504504,"ttft_p99_ms":1384.2828293819912,"tpot_p50_ms":41.9092545997052,"tpot_p95_ms":48.81261050516082,"tpot_p99_ms":52.42978161430527,"e2e_p50_ms":86774.6166390134,"e2e_p95_ms":100687.71062081214,"e2e_p99_ms":108062.53385416463,"itl_p50_ms":30.04166902974248,"itl_p95_ms":94.25195275107389,"itl_p99_ms":131.68032213036588,"validation_errors":[],"timestamp":"2026-07-17T17:46:03+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":62.242696,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":998.5768822168466,"request_tps":0.24034203502408205,"input_tps":246.11024386466002,"output_tps":492.22048772932004,"total_tps":738.3307315939801,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":450.9782859822735,"ttft_p95_ms":7408.641500258818,"ttft_p99_ms":8691.669157696888,"tpot_p50_ms":94.42066302833308,"tpot_p95_ms":107.56017656927577,"tpot_p99_ms":116.89547348379999,"e2e_p50_ms":194324.0639595315,"e2e_p95_ms":221427.19419910572,"e2e_p99_ms":239648.39002555466,"itl_p50_ms":80.37964064472666,"itl_p95_ms":150.87407550308848,"itl_p99_ms":245.1717850053683,"validation_errors":[],"timestamp":"2026-07-17T18:03:04+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":-31.359049,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":700.8377675570082,"request_tps":0.11414909941121654,"input_tps":116.88867779708573,"output_tps":467.55471118834294,"total_tps":584.4433889854287,"mean_input_tokens":1024.0,"mean_output_tokens":4096.0,"ttft_p50_ms":288.7477019103244,"ttft_p95_ms":2723.189539590385,"ttft_p99_ms":2724.9256797647104,"tpot_p50_ms":32.842243048709264,"tpot_p95_ms":36.472730307839235,"tpot_p99_ms":38.71758120981589,"e2e_p50_ms":134838.9821195742,"e2e_p95_ms":149641.9542316231,"e2e_p99_ms":158840.70349114944,"itl_p50_ms":24.362307662765186,"itl_p95_ms":72.10881218197751,"itl_p99_ms":104.22983951866627,"validation_errors":[],"timestamp":"2026-07-17T18:15:06+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":877.2158710220829,"request_tps":0.18239524076733468,"input_tps":186.77272654575071,"output_tps":747.0909061830029,"total_tps":933.8636327287535,"mean_input_tokens":1024.0,"mean_output_tokens":4096.0,"ttft_p50_ms":313.85099154431373,"ttft_p95_ms":1417.250323027838,"ttft_p99_ms":1419.979615684133,"tpot_p50_ms":40.59215980693948,"tpot_p95_ms":46.4335691594969,"tpot_p99_ms":49.336998789711664,"e2e_p50_ms":166747.97448143363,"e2e_p95_ms":190450.11723081113,"e2e_p99_ms":202322.11450884817,"itl_p50_ms":30.216291003550094,"itl_p95_ms":86.789655033499,"itl_p99_ms":117.55082814488571,"validation_errors":[],"timestamp":"2026-07-17T18:30:07+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":59.786842,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i1024_o4096_a1.log"} +{"timestamp":"2026-07-17T19:33:43+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":4096,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":2,"status":"FAILED","completed":0,"failed":240,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c48_i1024_o4096_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c48_i1024_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":85.73451679502614,"request_tps":0.9331130913266099,"input_tps":3822.031222073794,"output_tps":119.43847568980607,"total_tps":3941.4696977636004,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":1470.7278844434768,"ttft_p95_ms":9568.854463670865,"ttft_p99_ms":11798.592559006065,"tpot_p50_ms":114.35900654324777,"tpot_p95_ms":153.8244290071114,"tpot_p99_ms":172.61645189337938,"e2e_p50_ms":16570.26435213629,"e2e_p95_ms":24466.699651814994,"e2e_p99_ms":27479.300211130165,"itl_p50_ms":36.34669748134911,"itl_p95_ms":451.41349744517356,"itl_p99_ms":1156.9406865164638,"validation_errors":[],"timestamp":"2026-07-17T19:41:09+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":48.70448487694375,"request_tps":0.821279602916725,"input_tps":3363.9612535469055,"output_tps":105.1237891733408,"total_tps":3469.0850427202463,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":864.0570639399812,"ttft_p95_ms":4438.895414560099,"ttft_p99_ms":5320.836681961082,"tpot_p50_ms":64.49783391751878,"tpot_p95_ms":86.37649382017258,"tpot_p99_ms":100.60051819223426,"e2e_p50_ms":9647.042758530006,"e2e_p95_ms":13360.990816552654,"e2e_p99_ms":14474.491501573939,"itl_p50_ms":20.754211970294516,"itl_p95_ms":265.11732887011067,"itl_p99_ms":585.6110312805228,"validation_errors":[],"timestamp":"2026-07-17T19:42:18+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":15.760057132923976,"request_tps":0.3172577331305871,"input_tps":1299.4876749028847,"output_tps":40.60898984071515,"total_tps":1340.0966647435998,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":325.9119528811425,"ttft_p95_ms":673.023828631267,"ttft_p99_ms":673.1868727784604,"tpot_p50_ms":20.956370669717746,"tpot_p95_ms":22.818476094783644,"tpot_p99_ms":23.048002167465533,"e2e_p50_ms":3157.6877450570464,"e2e_p95_ms":3323.682710947469,"e2e_p99_ms":3331.6706885490566,"itl_p50_ms":15.452565314869085,"itl_p95_ms":53.95852787575367,"itl_p99_ms":83.0418958631339,"validation_errors":[],"timestamp":"2026-07-17T19:42:54+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":94.77433064510114,"request_tps":0.8441104195140541,"input_tps":3457.4762783295655,"output_tps":216.09226739559784,"total_tps":3673.5685457251634,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":1448.1975134694949,"ttft_p95_ms":2673.4470797237004,"ttft_p99_ms":3169.364200890992,"tpot_p50_ms":70.45431375120053,"tpot_p95_ms":91.22108132322775,"tpot_p99_ms":107.21685472348085,"e2e_p50_ms":19239.088561036624,"e2e_p95_ms":25206.23293695971,"e2e_p99_ms":28010.15280179446,"itl_p50_ms":25.20992630161345,"itl_p95_ms":322.23679299931973,"itl_p99_ms":644.4708909839392,"validation_errors":[],"timestamp":"2026-07-17T19:44:49+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":170.7196465348825,"request_tps":0.937209063207074,"input_tps":3838.808322896175,"output_tps":239.92552018101094,"total_tps":4078.7338430771856,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":2136.976893991232,"ttft_p95_ms":17337.70337848689,"ttft_p99_ms":21848.89752671588,"tpot_p50_ms":120.93104415883622,"tpot_p95_ms":169.59998784700917,"tpot_p99_ms":184.2107256766883,"e2e_p50_ms":33801.26656603534,"e2e_p95_ms":48432.99437782259,"e2e_p99_ms":54640.93346004141,"itl_p50_ms":33.9033316122368,"itl_p95_ms":432.9060880312075,"itl_p99_ms":1406.7125554041315,"validation_errors":[],"timestamp":"2026-07-17T19:48:01+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":11.029202,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c32_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c32_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":147.5610583620146,"request_tps":0.5421484562934914,"input_tps":2220.6400769781408,"output_tps":277.5800096222676,"total_tps":2498.220086600408,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":849.7430928982794,"ttft_p95_ms":8644.559909368394,"ttft_p99_ms":10933.468626951799,"tpot_p50_ms":52.187837612470744,"tpot_p95_ms":65.35931022151114,"tpot_p99_ms":72.77438619932695,"e2e_p50_ms":28306.824884959497,"e2e_p95_ms":36966.30423006136,"e2e_p99_ms":40133.17604351323,"itl_p50_ms":24.622750002890825,"itl_p95_ms":190.5647482490167,"itl_p99_ms":392.87617588182906,"validation_errors":[],"timestamp":"2026-07-17T19:50:50+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":105.00476745096967,"request_tps":0.3809350848634313,"input_tps":1560.3101076006146,"output_tps":195.03876345007683,"total_tps":1755.3488710506915,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":740.863966406323,"ttft_p95_ms":4399.81691209832,"ttft_p99_ms":5198.785737226717,"tpot_p50_ms":36.77453029351291,"tpot_p95_ms":45.51732738626712,"tpot_p99_ms":52.785640895483304,"e2e_p50_ms":19742.100903997198,"e2e_p95_ms":26249.77421091171,"e2e_p99_ms":28264.70110317925,"itl_p50_ms":20.399680283541482,"itl_p95_ms":92.81774598639458,"itl_p99_ms":288.86700700968504,"validation_errors":[],"timestamp":"2026-07-17T19:52:56+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":56.94767428189516,"request_tps":0.08779989811786928,"input_tps":359.6283826907926,"output_tps":44.953547836349074,"total_tps":404.58193052714165,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":673.970186850056,"ttft_p95_ms":825.8909878786653,"ttft_p99_ms":827.7391982730478,"tpot_p50_ms":20.820062029323086,"tpot_p95_ms":22.04351575531197,"tpot_p99_ms":22.143097754473093,"e2e_p50_ms":11313.021883834153,"e2e_p95_ms":12061.215797066689,"e2e_p99_ms":12137.079802453518,"itl_p50_ms":15.502929997940859,"itl_p95_ms":55.36682670159883,"itl_p99_ms":83.49743799772114,"validation_errors":[],"timestamp":"2026-07-17T19:54:13+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":230.4730543580372,"request_tps":0.3471121612148244,"input_tps":1421.7714123359208,"output_tps":355.4428530839802,"total_tps":1777.214265419901,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":751.6955588944256,"ttft_p95_ms":5875.389654969327,"ttft_p99_ms":8164.7384506091485,"tpot_p50_ms":42.576117347630024,"tpot_p95_ms":49.164676727369276,"tpot_p99_ms":55.02699510844375,"e2e_p50_ms":44724.99556443654,"e2e_p95_ms":54891.12991783767,"e2e_p99_ms":57256.77125822286,"itl_p50_ms":24.6061678432549,"itl_p95_ms":100.16557698448499,"itl_p99_ms":305.95273602132994,"validation_errors":[],"timestamp":"2026-07-17T19:58:24+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":175.8628815819975,"request_tps":0.22744992940053513,"input_tps":931.6349108245919,"output_tps":232.90872770614797,"total_tps":1164.5436385307398,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":721.6533330501989,"ttft_p95_ms":4550.052364449945,"ttft_p99_ms":5349.351275337394,"tpot_p50_ms":31.741651778545446,"tpot_p95_ms":36.02376814212655,"tpot_p99_ms":39.78750788387983,"e2e_p50_ms":33454.42393096164,"e2e_p95_ms":38895.44785992475,"e2e_p99_ms":41908.16472564125,"itl_p50_ms":20.35756300513943,"itl_p95_ms":70.59859298169613,"itl_p99_ms":228.45816581354785,"validation_errors":[],"timestamp":"2026-07-17T20:01:41+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":111.25815884512849,"request_tps":0.044940524379519946,"input_tps":184.0763878585137,"output_tps":46.019096964628424,"total_tps":230.09548482314213,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":672.2979180049151,"ttft_p95_ms":770.7921431865543,"ttft_p99_ms":787.7006725873798,"tpot_p50_ms":21.056790334306374,"tpot_p95_ms":22.361581890733653,"tpot_p99_ms":22.459885134368058,"e2e_p50_ms":22333.024316933006,"e2e_p95_ms":23550.986507860944,"e2e_p99_ms":23649.318473590538,"itl_p50_ms":15.539255303641161,"itl_p95_ms":55.356122887072495,"itl_p99_ms":88.48324394784868,"validation_errors":[],"timestamp":"2026-07-17T20:03:52+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":1024,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":403.99009139603004,"request_tps":0.19802465878198058,"input_tps":811.1090023709925,"output_tps":405.55450118549624,"total_tps":1216.6635035564886,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":735.552457976155,"ttft_p95_ms":5931.944755499713,"ttft_p99_ms":8281.503168595955,"tpot_p50_ms":36.951949444060766,"tpot_p95_ms":42.2074064977786,"tpot_p99_ms":45.32971975111284,"e2e_p50_ms":77803.16177441273,"e2e_p95_ms":89289.41085200058,"e2e_p99_ms":93578.7720635347,"itl_p50_ms":24.56039865501225,"itl_p95_ms":82.35741965472698,"itl_p99_ms":229.24311637567976,"validation_errors":[],"timestamp":"2026-07-17T20:10:57+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":320.93589431396686,"request_tps":0.12463548237726438,"input_tps":510.5069358172749,"output_tps":255.25346790863745,"total_tps":765.7604037259124,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":760.1207180414349,"ttft_p95_ms":4357.1449506096515,"ttft_p99_ms":5219.618475523312,"tpot_p50_ms":30.08599491183918,"tpot_p95_ms":33.55889662076107,"tpot_p99_ms":34.82547616983505,"e2e_p50_ms":62469.963374431245,"e2e_p95_ms":69734.61320172064,"e2e_p99_ms":71995.5238681403,"itl_p50_ms":20.356883682931464,"itl_p95_ms":64.30832933013637,"itl_p99_ms":104.83444494777396,"validation_errors":[],"timestamp":"2026-07-17T20:16:39+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":216.16745665995404,"request_tps":0.023130216163228197,"input_tps":94.7413654045827,"output_tps":47.37068270229135,"total_tps":142.11204810687406,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":673.9171028602868,"ttft_p95_ms":686.158444872126,"ttft_p99_ms":687.7665026392788,"tpot_p50_ms":20.552358145106258,"tpot_p95_ms":21.865023854045308,"tpot_p99_ms":21.873641400427378,"e2e_p50_ms":42744.5942258928,"e2e_p95_ms":45437.832057476044,"e2e_p99_ms":45453.86411715299,"itl_p50_ms":15.553842997178435,"itl_p95_ms":56.06750934384763,"itl_p99_ms":85.43721598107368,"validation_errors":[],"timestamp":"2026-07-17T20:20:35+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":769.710641612066,"request_tps":0.10393516170238944,"input_tps":425.71842233298713,"output_tps":425.71842233298713,"total_tps":851.4368446659743,"mean_input_tokens":4096.0,"mean_output_tokens":4096.0,"ttft_p50_ms":733.8229820597917,"ttft_p95_ms":6285.652554838449,"ttft_p99_ms":8608.823579458985,"tpot_p50_ms":35.110793266444254,"tpot_p95_ms":38.98668026696875,"tpot_p99_ms":43.160903312970305,"e2e_p50_ms":144571.92357245367,"e2e_p95_ms":162868.50160874892,"e2e_p99_ms":179169.88578920235,"itl_p50_ms":24.645808502100408,"itl_p95_ms":75.49262799633046,"itl_p99_ms":123.67290898691863,"validation_errors":[],"timestamp":"2026-07-17T20:33:46+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":614.0933780272026,"request_tps":0.06513667372297917,"input_tps":266.7998155693227,"output_tps":266.7998155693227,"total_tps":533.5996311386453,"mean_input_tokens":4096.0,"mean_output_tokens":4096.0,"ttft_p50_ms":772.6713895099238,"ttft_p95_ms":4500.128375564234,"ttft_p99_ms":5295.348061481491,"tpot_p50_ms":28.716161738374975,"tpot_p95_ms":31.966894855637495,"tpot_p99_ms":32.56942765887711,"e2e_p50_ms":119266.41606050543,"e2e_p95_ms":131652.94155926676,"e2e_p99_ms":134326.28932296298,"itl_p50_ms":20.384836631516617,"itl_p95_ms":63.262651674449444,"itl_p99_ms":95.26302095036954,"validation_errors":[],"timestamp":"2026-07-17T20:44:22+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":4096,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":434.6045183050446,"request_tps":0.011504712421076464,"input_tps":47.123302076729196,"output_tps":47.123302076729196,"total_tps":94.24660415345839,"mean_input_tokens":4096.0,"mean_output_tokens":4096.0,"ttft_p50_ms":676.5670769382268,"ttft_p95_ms":680.1287620794028,"ttft_p99_ms":680.2299901004881,"tpot_p50_ms":20.6595960995967,"tpot_p95_ms":22.141828008475958,"tpot_p99_ms":22.157888364503943,"e2e_p50_ms":85277.61310478672,"e2e_p95_ms":91350.53485170938,"e2e_p99_ms":91416.2007816229,"itl_p50_ms":15.550651354715228,"itl_p95_ms":57.56005547785505,"itl_p99_ms":88.97684107068926,"validation_errors":[],"timestamp":"2026-07-17T20:51:57+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":4096,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":262.53618311788887,"request_tps":0.3047199020337586,"input_tps":4992.530874921101,"output_tps":39.0041474603211,"total_tps":5031.535022381422,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":6358.780842041597,"ttft_p95_ms":28704.441011021834,"ttft_p99_ms":38341.95364549522,"tpot_p50_ms":352.26401160232814,"tpot_p95_ms":489.4379813777735,"tpot_p99_ms":593.2430667880972,"e2e_p50_ms":52364.81351137627,"e2e_p95_ms":83245.08615806699,"e2e_p99_ms":89560.93125953803,"itl_p50_ms":37.25044002446035,"itl_p95_ms":1611.4630710023146,"itl_p99_ms":5829.237112309784,"validation_errors":[],"timestamp":"2026-07-17T20:56:42+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":140.09482126194052,"request_tps":0.2855209039112909,"input_tps":4677.9744896825905,"output_tps":36.54667570064524,"total_tps":4714.521165383236,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":4229.60297355894,"ttft_p95_ms":15744.774443120687,"ttft_p99_ms":20490.973586495966,"tpot_p50_ms":177.7546691697267,"tpot_p95_ms":230.19704593338187,"tpot_p99_ms":292.3302190440457,"e2e_p50_ms":28359.413678990677,"e2e_p95_ms":40938.24249800527,"e2e_p99_ms":46125.68758594337,"itl_p50_ms":29.32046802015975,"itl_p95_ms":841.5840703120855,"itl_p99_ms":2737.0772002614103,"validation_errors":[],"timestamp":"2026-07-17T20:59:25+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":26.56212196405977,"request_tps":0.1882379731094269,"input_tps":3084.09095142485,"output_tps":24.09446055800664,"total_tps":3108.1854119828567,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":2880.8204801753163,"ttft_p95_ms":3176.355882314965,"ttft_p99_ms":3222.1149013843387,"tpot_p50_ms":22.557312661165913,"tpot_p95_ms":23.59471494243957,"tpot_p99_ms":23.799393874424414,"e2e_p50_ms":5742.904277984053,"e2e_p95_ms":6050.126017816365,"e2e_p99_ms":6064.834748357534,"itl_p50_ms":15.62649601449569,"itl_p95_ms":60.207899970312916,"itl_p99_ms":129.97808633372188,"validation_errors":[],"timestamp":"2026-07-17T21:00:14+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":286.97823332296684,"request_tps":0.278766786852324,"input_tps":4567.315035788476,"output_tps":71.36429743419494,"total_tps":4638.679333222672,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":5841.843549627811,"ttft_p95_ms":21703.17530462052,"ttft_p99_ms":31296.759539952014,"tpot_p50_ms":196.90109713724357,"tpot_p95_ms":286.549539692499,"tpot_p99_ms":302.26755278180445,"e2e_p50_ms":57872.11365252733,"e2e_p95_ms":80580.94614926957,"e2e_p99_ms":92631.12913521464,"itl_p50_ms":27.15283663322528,"itl_p95_ms":690.566053664467,"itl_p99_ms":3646.0466467154524,"validation_errors":[],"timestamp":"2026-07-17T21:05:23+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":159.3499606770929,"request_tps":0.2510198297510477,"input_tps":4112.708890641165,"output_tps":64.26107641626821,"total_tps":4176.969967057434,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":3169.008999480866,"ttft_p95_ms":15947.585682326453,"ttft_p99_ms":20661.607829078566,"tpot_p50_ms":102.50882065087995,"tpot_p95_ms":136.97829400937928,"tpot_p99_ms":183.48408388855918,"e2e_p50_ms":31314.968921942636,"e2e_p95_ms":43350.48841503446,"e2e_p99_ms":53466.01324624149,"itl_p50_ms":21.60122098090748,"itl_p95_ms":592.7520613186061,"itl_p99_ms":1634.4085613576074,"validation_errors":[],"timestamp":"2026-07-17T21:08:26+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":40.47361333016306,"request_tps":0.12353727746550708,"input_tps":2024.034753994868,"output_tps":31.625543031169812,"total_tps":2055.660297026038,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":2887.189506087452,"ttft_p95_ms":3261.679669795558,"ttft_p99_ms":3333.04980433546,"tpot_p50_ms":21.499996332853446,"tpot_p95_ms":23.93692009698819,"tpot_p99_ms":23.94511791106825,"e2e_p50_ms":8339.140377007425,"e2e_p95_ms":8965.311407670379,"e2e_p99_ms":8991.695408634841,"itl_p50_ms":15.620024021094045,"itl_p95_ms":56.270909340431295,"itl_p99_ms":85.31792947789653,"validation_errors":[],"timestamp":"2026-07-17T21:09:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":337.91137529397383,"request_tps":0.23674846675523173,"input_tps":3878.8868793177166,"output_tps":121.21521497867865,"total_tps":4000.1020942963955,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":3111.7780435597524,"ttft_p95_ms":21927.72211027331,"ttft_p99_ms":36676.38649743506,"tpot_p50_ms":118.51095876231818,"tpot_p95_ms":161.31014882768238,"tpot_p99_ms":176.25340365893123,"e2e_p50_ms":64247.78099998366,"e2e_p95_ms":92222.85833999047,"e2e_p99_ms":117732.16155912726,"itl_p50_ms":25.83940528954069,"itl_p95_ms":538.5441067628562,"itl_p99_ms":1662.8444464908325,"validation_errors":[],"timestamp":"2026-07-17T21:15:27+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":195.41697163600475,"request_tps":0.2046905121143028,"input_tps":3353.649350480737,"output_tps":104.80154220252304,"total_tps":3458.4508926832605,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":2975.5716496147215,"ttft_p95_ms":15955.639354838051,"ttft_p99_ms":20650.177005522415,"tpot_p50_ms":67.86122492951053,"tpot_p95_ms":84.00351084908196,"tpot_p99_ms":88.31705080019037,"e2e_p50_ms":38873.290556133725,"e2e_p95_ms":52227.369014907155,"e2e_p99_ms":56149.266425089445,"itl_p50_ms":20.94494504854083,"itl_p95_ms":97.76362804695958,"itl_p99_ms":946.778079494834,"validation_errors":[],"timestamp":"2026-07-17T21:19:06+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":67.1438225088641,"request_tps":0.07446701443516887,"input_tps":1220.0675645058068,"output_tps":38.12711139080646,"total_tps":1258.1946758966133,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":2881.187042919919,"ttft_p95_ms":3151.226846314966,"ttft_p99_ms":3203.403338007629,"tpot_p50_ms":21.89191394356132,"tpot_p95_ms":22.81137472182629,"tpot_p99_ms":22.98756816083613,"e2e_p50_ms":14087.625289103016,"e2e_p95_ms":14608.243441116065,"e2e_p99_ms":14649.249032121152,"itl_p50_ms":15.674249346678454,"itl_p95_ms":59.738073117720546,"itl_p99_ms":87.89702050853532,"validation_errors":[],"timestamp":"2026-07-17T21:20:36+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":448.71739268116653,"request_tps":0.17828593521188407,"input_tps":2921.0367625115086,"output_tps":182.5647976569693,"total_tps":3103.601560168478,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":3088.191010407172,"ttft_p95_ms":21836.158198781766,"ttft_p99_ms":37689.164795805555,"tpot_p50_ms":78.05551007323565,"tpot_p95_ms":111.69584850920182,"tpot_p99_ms":136.82766961175506,"e2e_p50_ms":83704.37322102953,"e2e_p95_ms":123751.99071946554,"e2e_p99_ms":168097.87159943013,"itl_p50_ms":25.61001619324088,"itl_p95_ms":330.62314824201167,"itl_p99_ms":924.2891655303538,"validation_errors":[],"timestamp":"2026-07-17T21:28:26+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":271.64864593790844,"request_tps":0.14724903141664444,"input_tps":2412.5281307303026,"output_tps":150.7830081706439,"total_tps":2563.3111389009464,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":2975.68096104078,"ttft_p95_ms":15765.321808110448,"ttft_p99_ms":20496.891892242707,"tpot_p50_ms":48.996134582617465,"tpot_p95_ms":55.18786179050639,"tpot_p99_ms":58.199572994750184,"e2e_p50_ms":53209.679587511346,"e2e_p95_ms":67266.07282647163,"e2e_p99_ms":71508.34642550442,"itl_p50_ms":20.887334054956835,"itl_p95_ms":86.56766859348863,"itl_p99_ms":639.3007660579557,"validation_errors":[],"timestamp":"2026-07-17T21:33:20+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":120.67551409010775,"request_tps":0.04143342614033968,"input_tps":678.8452538833253,"output_tps":42.42782836770783,"total_tps":721.2730822510332,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":2890.5434170737863,"ttft_p95_ms":3155.548330117017,"ttft_p99_ms":3207.341898921877,"tpot_p50_ms":20.92232840943204,"tpot_p95_ms":22.79761803845346,"tpot_p99_ms":22.93519996658317,"e2e_p50_ms":24283.944048918784,"e2e_p95_ms":26283.28570043668,"e2e_p99_ms":26372.238444108516,"itl_p50_ms":15.693355739737552,"itl_p95_ms":56.9020623806864,"itl_p99_ms":86.02622395614151,"validation_errors":[],"timestamp":"2026-07-17T21:35:44+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":1024,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":650.1041728160344,"request_tps":0.12305720120741064,"input_tps":2016.1691845822158,"output_tps":252.02114807277698,"total_tps":2268.1903326549927,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":3038.538462482393,"ttft_p95_ms":21865.582548070226,"ttft_p99_ms":37859.720940955296,"tpot_p50_ms":56.178174745273274,"tpot_p95_ms":75.81446139961089,"tpot_p99_ms":102.11669243301972,"e2e_p50_ms":118953.1502664322,"e2e_p95_ms":160926.5482754563,"e2e_p99_ms":234376.49603501422,"itl_p50_ms":25.71524027734995,"itl_p95_ms":93.14681623363867,"itl_p99_ms":662.986290517729,"validation_errors":[],"timestamp":"2026-07-17T21:46:55+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":427.34328094194643,"request_tps":0.09360156526114635,"input_tps":1533.5680452386218,"output_tps":191.69600565482773,"total_tps":1725.2640508934496,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2935.733180027455,"ttft_p95_ms":15794.045941426877,"ttft_p99_ms":20451.66814699536,"tpot_p50_ms":38.97004255910759,"tpot_p95_ms":44.13799750958157,"tpot_p99_ms":45.2419527712834,"e2e_p50_ms":82667.84297255799,"e2e_p95_ms":98572.18238646163,"e2e_p99_ms":102686.72488186043,"itl_p50_ms":20.94808127731085,"itl_p95_ms":68.98937463992637,"itl_p99_ms":443.4620583585153,"validation_errors":[],"timestamp":"2026-07-17T21:54:26+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":229.06133643817157,"request_tps":0.021828214563611455,"input_tps":357.6334674102101,"output_tps":44.70418342627626,"total_tps":402.3376508364864,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2877.515294123441,"ttft_p95_ms":3241.56024097465,"ttft_p99_ms":3283.969599334523,"tpot_p50_ms":20.819017763008898,"tpot_p95_ms":22.775988473179765,"tpot_p99_ms":22.928325767311733,"e2e_p50_ms":45494.04465500265,"e2e_p95_ms":49704.973551724106,"e2e_p99_ms":49974.39863445237,"itl_p50_ms":15.70363265151779,"itl_p95_ms":56.10076904607316,"itl_p99_ms":82.02209339709948,"validation_errors":[],"timestamp":"2026-07-17T21:58:38+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1171.3025146720465,"request_tps":0.06830003265415957,"input_tps":4476.110940023002,"output_tps":8.742404179732425,"total_tps":4484.853344202734,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":41774.10260157194,"ttft_p95_ms":154921.9547315384,"ttft_p99_ms":203285.24059475155,"tpot_p50_ms":1512.0516847589927,"tpot_p95_ms":2339.9997203555836,"tpot_p99_ms":2598.6683048809487,"e2e_p50_ms":236196.02562393993,"e2e_p95_ms":380147.26823875675,"e2e_p99_ms":411610.7790795639,"itl_p50_ms":38.77833753358573,"itl_p95_ms":9194.213955527326,"itl_p99_ms":29005.484325035166,"validation_errors":[],"timestamp":"2026-07-17T22:18:43+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":65536,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c16_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c16_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":585.3283604141325,"request_tps":0.0683377104292352,"input_tps":4478.580190690358,"output_tps":8.747226934942105,"total_tps":4487.3274176253,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":27782.567539601587,"ttft_p95_ms":73290.11640070235,"ttft_p99_ms":95623.88628913321,"tpot_p50_ms":709.3157277316016,"tpot_p95_ms":1171.9882496466719,"tpot_p99_ms":1284.1396156144988,"e2e_p50_ms":118592.42226043716,"e2e_p95_ms":176901.89834813352,"e2e_p99_ms":195259.67974432508,"itl_p50_ms":26.556531044964988,"itl_p95_ms":4446.808243403211,"itl_p99_ms":14378.40555771254,"validation_errors":[],"timestamp":"2026-07-17T22:29:04+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":65536,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c8_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c8_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":72.90795596619137,"request_tps":0.06857962116395888,"input_tps":4494.434052601209,"output_tps":8.778191508986737,"total_tps":4503.2122441101965,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":14517.88427401334,"ttft_p95_ms":14806.680322438478,"ttft_p99_ms":14857.269734106958,"tpot_p50_ms":22.28340941598624,"tpot_p95_ms":23.089399559452662,"tpot_p99_ms":23.14871986044615,"e2e_p50_ms":17347.877269843593,"e2e_p95_ms":17739.034066488966,"e2e_p99_ms":17797.15715638362,"itl_p50_ms":15.680385986343026,"itl_p95_ms":60.68944364475707,"itl_p99_ms":88.50785356480628,"validation_errors":[],"timestamp":"2026-07-17T22:30:51+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":65536,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/raw_outputs/sglang_adaptive_c1_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/tp8_dp1/logs/sglang_c1_i65536_o128_a1.log"} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_shapes.jsonl new file mode 100644 index 0000000..b7a5fa8 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_shapes.jsonl @@ -0,0 +1,36 @@ +{"timestamp":"2026-07-17T08:00:37+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":512,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":2372.1410847965403,"last_total_tps":1852.4569382744771} +{"timestamp":"2026-07-17T08:13:13+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":1805.8164621293636,"last_total_tps":311.23570174130094} +{"timestamp":"2026-07-17T08:37:44+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":1281.951554285542,"last_total_tps":184.27954639404544} +{"timestamp":"2026-07-17T09:45:05+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":1024,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":2,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":924.3769547408627,"last_total_tps":924.3769547408627} +{"timestamp":"2026-07-17T10:01:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":512,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":713.301610156369,"last_total_tps":415.35293151931256} +{"timestamp":"2026-07-17T11:17:07+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":2,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":613.9955200884395,"last_total_tps":613.9955200884395} +{"timestamp":"2026-07-17T11:27:55+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":4055.189542442062,"last_total_tps":1087.6941389647598} +{"timestamp":"2026-07-17T11:40:23+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":897.4416357663325,"last_total_tps":651.9774108402153} +{"timestamp":"2026-07-17T11:52:18+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":1352.2428521060174,"last_total_tps":383.4246271325376} +{"timestamp":"2026-07-17T12:09:52+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":900.1024546224579,"last_total_tps":216.7499072532702} +{"timestamp":"2026-07-17T13:20:40+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":659.227133681341,"last_total_tps":133.1680857505817} +{"timestamp":"2026-07-17T14:26:47+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":4096,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":1,"search_cap":512,"max_successful_concurrency":0,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":null,"best_total_tps":null,"last_total_tps":null} +{"timestamp":"2026-07-17T14:42:42+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":5125.8042771894325,"last_total_tps":2230.3585086450407} +{"timestamp":"2026-07-17T14:59:25+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":3694.7511429510405,"last_total_tps":1632.8138573130561} +{"timestamp":"2026-07-17T15:13:20+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":3114.4132353131213,"last_total_tps":1057.6609699448902} +{"timestamp":"2026-07-17T15:40:04+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":1936.7830475274534,"last_total_tps":630.9061324181441} +{"timestamp":"2026-07-17T16:20:45+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":16384,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":1204.231632640546,"last_total_tps":364.32863368782785} +{"timestamp":"2026-07-17T16:48:01+00:00","engine":"sglang","tp":4,"dp":2,"mark":"Y","isl":65536,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_GROUP_SKIP_THRESHOLD","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":6150.9700433587495,"last_total_tps":2913.5583338875285} +{"timestamp":"2026-07-17T16:58:14+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":3450.755550865165,"last_total_tps":2532.7652735816637} +{"timestamp":"2026-07-17T17:04:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":2587.6325571949974,"last_total_tps":1795.9101973285085} +{"timestamp":"2026-07-17T17:14:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":1798.7457708915072,"last_total_tps":1266.168984578888} +{"timestamp":"2026-07-17T17:31:33+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":1379.2244951274456,"last_total_tps":931.9574879084522} +{"timestamp":"2026-07-17T18:03:04+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":1075.6417634494053,"last_total_tps":738.3307315939801} +{"timestamp":"2026-07-17T19:33:43+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":3,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":32,"best_total_tps":933.8636327287535,"last_total_tps":933.8636327287535} +{"timestamp":"2026-07-17T19:42:54+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":3941.4696977636004,"last_total_tps":1340.0966647435998} +{"timestamp":"2026-07-17T19:48:01+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":4078.7338430771856,"last_total_tps":4078.7338430771856} +{"timestamp":"2026-07-17T19:54:13+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":2498.220086600408,"last_total_tps":404.58193052714165} +{"timestamp":"2026-07-17T20:03:52+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":1777.214265419901,"last_total_tps":230.09548482314213} +{"timestamp":"2026-07-17T20:20:35+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":1216.6635035564886,"last_total_tps":142.11204810687406} +{"timestamp":"2026-07-17T20:51:57+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":4096,"osl":4096,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":851.4368446659743,"last_total_tps":94.24660415345839} +{"timestamp":"2026-07-17T21:00:14+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":5031.535022381422,"last_total_tps":3108.1854119828567} +{"timestamp":"2026-07-17T21:09:29+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":4638.679333222672,"last_total_tps":2055.660297026038} +{"timestamp":"2026-07-17T21:20:36+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":4000.1020942963955,"last_total_tps":1258.1946758966133} +{"timestamp":"2026-07-17T21:35:44+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":3103.601560168478,"last_total_tps":721.2730822510332} +{"timestamp":"2026-07-17T21:58:38+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":16384,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":2268.1903326549927,"last_total_tps":402.3376508364864} +{"timestamp":"2026-07-17T22:30:51+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":65536,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_GROUP_SKIP_THRESHOLD","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":1,"best_total_tps":4503.2122441101965,"last_total_tps":4503.2122441101965} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.jsonl new file mode 100644 index 0000000..078efac --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.jsonl @@ -0,0 +1,36 @@ +{"timestamp": "2026-07-17T08:00:37+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 8, "saturation_concurrency": null, "stop_probe_concurrency": 8, "best_tps_concurrency": 16, "best_total_tps": 2372.1410847965403, "last_total_tps": 1852.4569382744771} +{"timestamp": "2026-07-17T08:13:13+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 32, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 16, "best_total_tps": 1805.8164621293636, "last_total_tps": 311.23570174130094} +{"timestamp": "2026-07-17T08:37:44+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 32, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 16, "best_total_tps": 1281.951554285542, "last_total_tps": 184.27954639404544} +{"timestamp": "2026-07-17T09:45:05+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 1024, "status": "FAILED", "stop_reason": "BENCHMARK_FAILED", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 16, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 16, "best_total_tps": 924.3769547408627, "last_total_tps": 924.3769547408627} +{"timestamp": "2026-07-17T10:01:07+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 8, "saturation_concurrency": null, "stop_probe_concurrency": 8, "best_tps_concurrency": 16, "best_total_tps": 713.301610156369, "last_total_tps": 415.35293151931256} +{"timestamp": "2026-07-17T11:17:07+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 1024, "osl": 4096, "status": "FAILED", "stop_reason": "BENCHMARK_FAILED", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 16, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 16, "best_total_tps": 613.9955200884395, "last_total_tps": 613.9955200884395} +{"timestamp": "2026-07-17T11:27:55+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 4055.189542442062, "last_total_tps": 1087.6941389647598} +{"timestamp": "2026-07-17T11:40:23+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 897.4416357663325, "last_total_tps": 651.9774108402153} +{"timestamp": "2026-07-17T11:52:18+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 1352.2428521060174, "last_total_tps": 383.4246271325376} +{"timestamp": "2026-07-17T12:09:52+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 1024, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 900.1024546224579, "last_total_tps": 216.7499072532702} +{"timestamp": "2026-07-17T13:20:40+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 659.227133681341, "last_total_tps": 133.1680857505817} +{"timestamp": "2026-07-17T14:26:47+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 4096, "osl": 4096, "status": "FAILED", "stop_reason": "BENCHMARK_FAILED", "tested_points": 1, "search_cap": 512, "max_successful_concurrency": 0, "saturation_concurrency": null, "stop_probe_concurrency": 16, "best_tps_concurrency": null, "best_total_tps": null, "last_total_tps": null} +{"timestamp": "2026-07-17T14:42:42+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 16384, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 5125.8042771894325, "last_total_tps": 2230.3585086450407} +{"timestamp": "2026-07-17T14:59:25+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 16384, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 3694.7511429510405, "last_total_tps": 1632.8138573130561} +{"timestamp": "2026-07-17T15:13:20+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 16384, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 3114.4132353131213, "last_total_tps": 1057.6609699448902} +{"timestamp": "2026-07-17T15:40:04+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 16384, "osl": 1024, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 1936.7830475274534, "last_total_tps": 630.9061324181441} +{"timestamp": "2026-07-17T16:20:45+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 16384, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 1204.231632640546, "last_total_tps": 364.32863368782785} +{"timestamp": "2026-07-17T16:48:01+00:00", "engine": "sglang", "tp": 4, "dp": 2, "mark": "Y", "isl": 65536, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_GROUP_SKIP_THRESHOLD", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 8, "best_total_tps": 6150.9700433587495, "last_total_tps": 2913.5583338875285} +{"timestamp": "2026-07-17T16:58:14+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 48, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 3450.755550865165, "last_total_tps": 2532.7652735816637} +{"timestamp": "2026-07-17T17:04:29+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 48, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 2587.6325571949974, "last_total_tps": 1795.9101973285085} +{"timestamp": "2026-07-17T17:14:29+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 48, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 1798.7457708915072, "last_total_tps": 1266.168984578888} +{"timestamp": "2026-07-17T17:31:33+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 1024, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 48, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 1379.2244951274456, "last_total_tps": 931.9574879084522} +{"timestamp": "2026-07-17T18:03:04+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 48, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 1075.6417634494053, "last_total_tps": 738.3307315939801} +{"timestamp": "2026-07-17T19:33:43+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 4096, "status": "FAILED", "stop_reason": "BENCHMARK_FAILED", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 32, "saturation_concurrency": null, "stop_probe_concurrency": 48, "best_tps_concurrency": 32, "best_total_tps": 933.8636327287535, "last_total_tps": 933.8636327287535} +{"timestamp": "2026-07-17T19:42:54+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 3941.4696977636004, "last_total_tps": 1340.0966647435998} +{"timestamp": "2026-07-17T19:48:01+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_EXCEEDED", "tested_points": 2, "search_cap": 512, "max_successful_concurrency": 32, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 32, "best_total_tps": 4078.7338430771856, "last_total_tps": 4078.7338430771856} +{"timestamp": "2026-07-17T19:54:13+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 2498.220086600408, "last_total_tps": 404.58193052714165} +{"timestamp": "2026-07-17T20:03:52+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 1024, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 1777.214265419901, "last_total_tps": 230.09548482314213} +{"timestamp": "2026-07-17T20:20:35+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 1216.6635035564886, "last_total_tps": 142.11204810687406} +{"timestamp": "2026-07-17T20:51:57+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 4096, "osl": 4096, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 851.4368446659743, "last_total_tps": 94.24660415345839} +{"timestamp": "2026-07-17T21:00:14+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 16384, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 5031.535022381422, "last_total_tps": 3108.1854119828567} +{"timestamp": "2026-07-17T21:09:29+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 16384, "osl": 256, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 4638.679333222672, "last_total_tps": 2055.660297026038} +{"timestamp": "2026-07-17T21:20:36+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 16384, "osl": 512, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 4000.1020942963955, "last_total_tps": 1258.1946758966133} +{"timestamp": "2026-07-17T21:35:44+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 16384, "osl": 1024, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 3103.601560168478, "last_total_tps": 721.2730822510332} +{"timestamp": "2026-07-17T21:58:38+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 16384, "osl": 2048, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_SLO_BACKOFF_FOUND", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 16, "best_total_tps": 2268.1903326549927, "last_total_tps": 402.3376508364864} +{"timestamp": "2026-07-17T22:30:51+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 65536, "osl": 128, "status": "TTFT_BOUNDARY", "stop_reason": "TTFT_GROUP_SKIP_THRESHOLD", "tested_points": 3, "search_cap": 512, "max_successful_concurrency": 1, "saturation_concurrency": null, "stop_probe_concurrency": 1, "best_tps_concurrency": 1, "best_total_tps": 4503.2122441101965, "last_total_tps": 4503.2122441101965} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.md b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.md new file mode 100644 index 0000000..2e73260 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/adaptive_summary.md @@ -0,0 +1,42 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| +| sglang | 4 | 2 | 1024 | 128 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 2372.1410847965403 | 8 | +| sglang | 4 | 2 | 1024 | 256 | TTFT_SLO_EXCEEDED | None | 16 | 1805.8164621293636 | 32 | +| sglang | 4 | 2 | 1024 | 512 | TTFT_SLO_EXCEEDED | None | 16 | 1281.951554285542 | 32 | +| sglang | 4 | 2 | 1024 | 1024 | BENCHMARK_FAILED | None | 16 | 924.3769547408627 | 16 | +| sglang | 4 | 2 | 1024 | 2048 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 713.301610156369 | 8 | +| sglang | 4 | 2 | 1024 | 4096 | BENCHMARK_FAILED | None | 16 | 613.9955200884395 | 16 | +| sglang | 4 | 2 | 4096 | 128 | TTFT_SLO_BACKOFF_FOUND | None | 8 | 4055.189542442062 | 1 | +| sglang | 4 | 2 | 4096 | 256 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 897.4416357663325 | 1 | +| sglang | 4 | 2 | 4096 | 512 | TTFT_SLO_BACKOFF_FOUND | None | 8 | 1352.2428521060174 | 1 | +| sglang | 4 | 2 | 4096 | 1024 | TTFT_SLO_BACKOFF_FOUND | None | 8 | 900.1024546224579 | 1 | +| sglang | 4 | 2 | 4096 | 2048 | TTFT_SLO_BACKOFF_FOUND | None | 8 | 659.227133681341 | 1 | +| sglang | 4 | 2 | 4096 | 4096 | BENCHMARK_FAILED | None | None | None | 0 | +| sglang | 4 | 2 | 16384 | 128 | TTFT_SLO_EXCEEDED | None | 8 | 5125.8042771894325 | 1 | +| sglang | 4 | 2 | 16384 | 256 | TTFT_SLO_EXCEEDED | None | 16 | 3694.7511429510405 | 1 | +| sglang | 4 | 2 | 16384 | 512 | TTFT_SLO_EXCEEDED | None | 8 | 3114.4132353131213 | 1 | +| sglang | 4 | 2 | 16384 | 1024 | TTFT_SLO_EXCEEDED | None | 8 | 1936.7830475274534 | 1 | +| sglang | 4 | 2 | 16384 | 2048 | TTFT_SLO_EXCEEDED | None | 8 | 1204.231632640546 | 1 | +| sglang | 4 | 2 | 65536 | 128 | TTFT_GROUP_SKIP_THRESHOLD | None | 8 | 6150.9700433587495 | 1 | +| sglang | 8 | 1 | 1024 | 128 | TTFT_SLO_EXCEEDED | None | 32 | 3450.755550865165 | 48 | +| sglang | 8 | 1 | 1024 | 256 | TTFT_SLO_EXCEEDED | None | 32 | 2587.6325571949974 | 48 | +| sglang | 8 | 1 | 1024 | 512 | TTFT_SLO_EXCEEDED | None | 32 | 1798.7457708915072 | 48 | +| sglang | 8 | 1 | 1024 | 1024 | TTFT_SLO_EXCEEDED | None | 32 | 1379.2244951274456 | 48 | +| sglang | 8 | 1 | 1024 | 2048 | TTFT_SLO_EXCEEDED | None | 32 | 1075.6417634494053 | 48 | +| sglang | 8 | 1 | 1024 | 4096 | BENCHMARK_FAILED | None | 32 | 933.8636327287535 | 32 | +| sglang | 8 | 1 | 4096 | 128 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 3941.4696977636004 | 1 | +| sglang | 8 | 1 | 4096 | 256 | TTFT_SLO_EXCEEDED | None | 32 | 4078.7338430771856 | 32 | +| sglang | 8 | 1 | 4096 | 512 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 2498.220086600408 | 1 | +| sglang | 8 | 1 | 4096 | 1024 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 1777.214265419901 | 1 | +| sglang | 8 | 1 | 4096 | 2048 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 1216.6635035564886 | 1 | +| sglang | 8 | 1 | 4096 | 4096 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 851.4368446659743 | 1 | +| sglang | 8 | 1 | 16384 | 128 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 5031.535022381422 | 1 | +| sglang | 8 | 1 | 16384 | 256 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 4638.679333222672 | 1 | +| sglang | 8 | 1 | 16384 | 512 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 4000.1020942963955 | 1 | +| sglang | 8 | 1 | 16384 | 1024 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 3103.601560168478 | 1 | +| sglang | 8 | 1 | 16384 | 2048 | TTFT_SLO_BACKOFF_FOUND | None | 16 | 2268.1903326549927 | 1 | +| sglang | 8 | 1 | 65536 | 128 | TTFT_GROUP_SKIP_THRESHOLD | None | 1 | 4503.2122441101965 | 1 | + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/run_manifest.json new file mode 100644 index 0000000..6a8a8d4 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "adaptive_20260717-075359", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/adaptive_20260717-075359/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_points.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_points.jsonl new file mode 100644 index 0000000..3f40b4f --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_points.jsonl @@ -0,0 +1,2 @@ +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":46.96696303598583,"request_tps":1.7033249507468566,"input_tps":1744.2047495647812,"output_tps":218.02559369559765,"total_tps":1962.230343260379,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":494.9538300279528,"ttft_p95_ms":3561.9220167398453,"ttft_p99_ms":3668.1022071442567,"tpot_p50_ms":58.32520933128806,"tpot_p95_ms":94.1695388358849,"tpot_p99_ms":107.79328334271742,"e2e_p50_ms":8111.887397128157,"e2e_p95_ms":15186.770990001969,"e2e_p99_ms":16331.26907782862,"itl_p50_ms":34.82708218507469,"itl_p95_ms":185.14514567020038,"itl_p99_ms":483.97617518901825,"validation_errors":[],"timestamp":"2026-07-17T07:27:47+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/tp8_dp1/raw_outputs/sglang_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/tp8_dp1/logs/sglang_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":51.43052648403682,"request_tps":3.110992846819512,"input_tps":3185.65667514318,"output_tps":398.2070843928975,"total_tps":3583.863759536078,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":556.1951335985214,"ttft_p95_ms":1466.6377054760233,"ttft_p99_ms":1469.2289810441434,"tpot_p50_ms":75.4069922121579,"tpot_p95_ms":104.81830453133489,"tpot_p99_ms":113.73199667007931,"e2e_p50_ms":10190.570478094742,"e2e_p95_ms":13657.75836262619,"e2e_p99_ms":14968.806141340172,"itl_p50_ms":42.77814150555059,"itl_p95_ms":228.68744850275107,"itl_p99_ms":434.5130304887425,"validation_errors":[],"timestamp":"2026-07-17T07:28:59+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":82.642358,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/tp8_dp1/raw_outputs/sglang_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/tp8_dp1/logs/sglang_c32_i1024_o128_a1.log"} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_shapes.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_shapes.jsonl new file mode 100644 index 0000000..4291e5c --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_shapes.jsonl @@ -0,0 +1 @@ +{"timestamp":"2026-07-17T07:28:59+00:00","engine":"sglang","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"status":"COMPLETED","stop_reason":"SEARCH_CAP_REACHED","tested_points":2,"search_cap":32,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":3583.863759536078,"last_total_tps":3583.863759536078} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.jsonl b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.jsonl new file mode 100644 index 0000000..962a6ac --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.jsonl @@ -0,0 +1 @@ +{"timestamp": "2026-07-17T07:28:59+00:00", "engine": "sglang", "tp": 8, "dp": 1, "mark": "Y", "isl": 1024, "osl": 128, "status": "COMPLETED", "stop_reason": "SEARCH_CAP_REACHED", "tested_points": 2, "search_cap": 32, "max_successful_concurrency": 32, "saturation_concurrency": null, "stop_probe_concurrency": 32, "best_tps_concurrency": 32, "best_total_tps": 3583.863759536078, "last_total_tps": 3583.863759536078} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.md b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.md new file mode 100644 index 0000000..00c46f1 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/adaptive_summary.md @@ -0,0 +1,7 @@ +# Adaptive concurrency search summary + +| Engine | TP | DP | ISL | OSL | Stop | Saturation C | Best TPS C | Best Total TPS | Max successful C | +|---|---:|---:|---:|---:|---|---:|---:|---:|---:| +| sglang | 8 | 1 | 1024 | 128 | SEARCH_CAP_REACHED | None | 32 | 3583.863759536078 | 32 | + +`Saturation C` is the first point in the final low-gain streak. `Best TPS C` is the tested point with the highest observed Total TPS. diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/run_manifest.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/run_manifest.json new file mode 100644 index 0000000..a7d1669 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "dsv4_p800_sglang_tp_dp_official", + "engine": "sglang", + "run_id": "smoke-20260717-072308", + "model": "/data1/models/DeepSeek-V4-Flash-INT8", + "hardware": "8x Kunlun P800 XPU", + "matrix": "/data1/yy/sskj/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 32, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/shapes.tsv b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/shapes.tsv new file mode 100644 index 0000000..f993a46 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/adaptive_results/smoke-20260717-072308/shapes.tsv @@ -0,0 +1,25 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/compare.py b/experiments/p800/dsv4_p800_sglang_tp_dp_official/compare.py new file mode 100644 index 0000000..0d68bd3 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/compare.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_p800_sglang_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom" or "metrics" not in s: + skipped[key][label] = s.get("note") or s.get("status", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: SGLang (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | XPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, osl = re.findall(r"\d+", scenario_name) + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/OSL. + f.write("\n## Best throughput per (ISL, OSL)\n\n") + f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, osl = re.findall(r"\d+", scenario_name) + isl_i, osl_i = int(isl), int(osl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]: + best_by_shape[(isl_i, osl_i)] = (out_tok, label, s) + for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env b/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env new file mode 100644 index 0000000..2342169 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/config.env @@ -0,0 +1,102 @@ +# TP×DP matrix experiment for DeepSeek-V4-Flash-INT8 on Kunlun P800 (8 XPUs) +# using SGLang-XPU with the OFFICIAL vendor-recommended launch parameters +# (extracted from the image's /workspace/ds_v4/w8a8/run_server.sh): +# --ep-size (expert parallel; probes whether EP makes TP2/DP4 viable) +# EAGLE speculative decoding (steps=3, topk=1, draft=4) +# --chunked-prefill-size 8192 --max-prefill-tokens 16384 +# --max-running-requests 64 +# Tests SGLang with three parallel configurations: +# TP=2, DP=4 -> 2 XPUs per replica, 4 replicas +# TP=4, DP=2 -> 4 XPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 XPUs, no data parallelism (production config) + +EXPERIMENT="dsv4_p800_sglang_tp_dp_official" +MODEL_NAME="DeepSeek-V4-Flash-INT8" +MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}" +SERVED_MODEL_NAME="deepseek-v4-flash-int8" + +SGLANG_PORT="${SGLANG_PORT:-30014}" + +# Dedicated container name so this experiment never touches the +# `sglang-dsv4-flash` container used by the other P800 experiments. +CONTAINER_NAME="sglang-dsv4-flash-tpdp-official" + +# Python interpreter inside the P800 SGLang container (used to run the +# benchmark client via `docker exec`). +CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}" + +export XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +# TP2/DP4 is excluded: --ep-size 8 crashes this vendor image when DP>1 +# (ZeroDivisionError in data_parallel_controller.py), and --ep-size 2 offers +# no memory relief (~264/2=132 GiB > 96 GiB per XPU), so TP=2 remains +# infeasible for this INT8 model on P800 regardless of EP. +# --ep-size is only passed when DP=1 (official production config TP8/EP8). +# Override with a space-separated list of comma pairs, e.g. +# PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1. +if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then + declare -a PARALLEL_CONFIGS=() + for pair in $PARALLEL_CONFIGS_STR; do + PARALLEL_CONFIGS+=("${pair//,/ }") + done +else + declare -a PARALLEL_CONFIGS=( + "4 2" + "8 1" + ) +fi + +# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so +# the context length is capped well below the H20 1M setting. +CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}" +MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}" + +# Official w8a8 parameters (from /workspace/ds_v4/w8a8/run_server.sh). +# EP_SIZE defaults to TP*DP in start_sglang_docker.sh (8 on this machine); +# override here only for single-config probes. +EP_SIZE="${EP_SIZE:-}" +CHUNKED_PREFILL_SIZE="${CHUNKED_PREFILL_SIZE:-8192}" +MAX_PREFILL_TOKENS="${MAX_PREFILL_TOKENS:-16384}" +MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}" +# EAGLE speculative decoding (official defaults: steps=3, topk=1, draft=4). +SPEC_NUM_STEPS="${SPEC_NUM_STEPS:-3}" +SPEC_EAGLE_TOPK="${SPEC_EAGLE_TOPK:-1}" +SPEC_NUM_DRAFT_TOKENS="${SPEC_NUM_DRAFT_TOKENS:-4}" + +DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}" + +# ShareGPT seed dataset for the random workload. DATASET_PATH is the host +# copy (also used by the adaptive search for capacity checks); it is mounted +# into the container at CONTAINER_DATASET_PATH, which is what the benchmark +# client references. Default matches the H20 experiments; the file is +# gitignored, download it into /data1/yy/sskj/datasets/ first. +DATASET_PATH="${DATASET_PATH:-/data1/yy/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" +CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency (kept for parity with the H20 experiment; +# generate_scenarios.py currently samples the low/high endpoints only). +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# XPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without +# starting any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated +# scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/generate_scenarios.py b/experiments/p800/dsv4_p800_sglang_tp_dp_official/generate_scenarios.py new file mode 100644 index 0000000..2bfb152 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + osl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for osl_str in sorted(osl_map.keys(), key=int): + mark = osl_map[osl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(osl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json b/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json new file mode 100644 index 0000000..a0b8099 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/matrix.json @@ -0,0 +1,54 @@ +{ + "comment": "ISL/OSL matrix for dsv4_p800_sglang_tp_dp_matrix. Y=must test, P=optional (record skipped on failure), N=skip.", + "mode": "Y", + "notes": "P800 INT8 sustains at most ~131k input tokens, so ISL stops at 131072. Concurrency ranges match the H20 matrix; scenarios that exceed server capacity are recorded as failed/skipped and the run moves on.", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { "low": 1, "high": 128 }, + "4096": { "low": 1, "high": 64 }, + "16384": { "low": 1, "high": 32 }, + "65536": { "low": 1, "high": 8 }, + "131072": { "low": 1, "high": 4 } + } +} diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/patches/bench_serving.py b/experiments/p800/dsv4_p800_sglang_tp_dp_official/patches/bench_serving.py new file mode 100644 index 0000000..97c752f --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/patches/bench_serving.py @@ -0,0 +1,2366 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +""" + +import argparse +import asyncio +import copy +import importlib.util +import json +import os +import random +import shutil +import sys +import time +import traceback +import uuid +import warnings +from argparse import ArgumentParser +from copy import deepcopy +from dataclasses import dataclass, field, replace +from datetime import datetime +from pathlib import Path +from typing import Any, AsyncGenerator, Callable, Dict, List, Optional, Tuple, Union + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm +from transformers import AutoTokenizer, PreTrainedTokenizerBase + +from sglang.benchmark.datasets import DatasetRow, get_dataset +from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time +from sglang.benchmark.utils import ( + get_tokenizer, + parse_custom_headers, + remove_prefix, + set_ulimit, +) +from sglang.srt.utils.network import NetworkAddress + +_ROUTING_KEY_HEADER = "X-SMG-Routing-Key" + +_EMBEDDING_UNSUPPORTED_DATASETS = {"image", "mmmu", "mooncake"} + +TERM_PLOTLIB_AVAILABLE = (importlib.util.find_spec("termplotlib") is not None) and ( + shutil.which("gnuplot") is not None +) + +global args + + +# don't want to import sglang package here +def _get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def _create_bench_client_session(): + # When the pressure is big, the read buffer could be full before aio thread read + # the content. We increase the read_bufsize from 64K to 10M. + # Define constants for timeout and buffer size for clarity and maintainability + BENCH_AIOHTTP_TIMEOUT_SECONDS = 6 * 60 * 60 # 6 hours + BENCH_AIOHTTP_READ_BUFSIZE_BYTES = 10 * 1024**2 # 10 MB + + aiohttp_timeout = aiohttp.ClientTimeout(total=BENCH_AIOHTTP_TIMEOUT_SECONDS) + return aiohttp.ClientSession( + timeout=aiohttp_timeout, read_bufsize=BENCH_AIOHTTP_READ_BUFSIZE_BYTES + ) + + +@dataclass +class RequestFuncInput: + prompt: Union[str, List[str], List[Dict[str, str]]] + api_url: str + prompt_len: int + output_len: int + model: str + lora_name: str + image_data: Optional[List[str]] + extra_request_body: Dict[str, Any] + timestamp: Optional[float] = None + routing_key: Optional[str] = None + + +@dataclass +class RequestFuncOutput: + generated_text: str = "" + success: bool = False + latency: float = 0.0 + ttft: float = 0.0 # Time to first token + itl: List[float] = field(default_factory=list) # List of inter-token latencies + text_chunks: List[str] = field(default_factory=list) + prompt_len: int = 0 + error: str = "" + output_len: int = 0 + start_time: float = 0.0 + + @staticmethod + def init_new(request_func_input: RequestFuncInput): + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + return output + + +def get_auth_headers() -> Dict[str, str]: + openai_api_key = os.environ.get("OPENAI_API_KEY") + if openai_api_key: + return {"Authorization": f"Bearer {openai_api_key}"} + else: + api_key = os.environ.get("API_KEY") + if api_key: + return {"Authorization": f"{api_key}"} + return {} + + +def get_request_headers() -> Dict[str, str]: + headers = get_auth_headers() + if h := getattr(args, "header", None): + headers.update(parse_custom_headers(h)) + return headers + + +def wait_for_endpoint(url: str, timeout_sec: int = 60) -> bool: + """Wait for the server to become ready by polling the given URL.""" + print(f"Waiting up to {timeout_sec}s for {url} to become ready...") + start_time = time.perf_counter() + headers = get_auth_headers() + while True: + try: + response = requests.get(url, headers=headers, timeout=5) + if response.status_code == 200: + elapsed = time.perf_counter() - start_time + print(f"Server ready in {elapsed:.1f}s.") + return True + except requests.exceptions.RequestException: + pass + elapsed = time.perf_counter() - start_time + if elapsed >= timeout_sec: + print(f"Server did not become ready within {timeout_sec}s timeout.") + return False + time.sleep(1) + + +# trt llm does not support ignore_eos +# https://github.com/triton-inference-server/tensorrtllm_backend/issues/505 +async def async_request_trt_llm( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith("generate_stream") + + async with _create_bench_client_session() as session: + payload = { + "accumulate_tokens": True, + "text_input": request_func_input.prompt, + "temperature": 0.000001, + "top_p": 1.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "min_length": request_func_input.output_len, + "end_id": 1048576, + **request_func_input.extra_request_body, + } + if args.disable_ignore_eos: + del payload["min_length"] + del payload["end_id"] + output = RequestFuncOutput.init_new(request_func_input) + + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + + data = json.loads(chunk) + output.generated_text += data["text_output"] + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + + output.latency = most_recent_timestamp - st + output.success = True + output.output_len = request_func_input.output_len + + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + # Build payload with defaults that can be overridden by extra_request_body + payload = { + "model": request_func_input.model, + "prompt": prompt, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + } + + # Add temperature default only if not specified in extra_request_body + if "temperature" not in request_func_input.extra_request_body: + payload["temperature"] = 0.0 + + # Add ignore_eos default only if not specified in extra_request_body + if "ignore_eos" not in request_func_input.extra_request_body: + payload["ignore_eos"] = not args.disable_ignore_eos + + if args.return_logprob and args.top_logprobs_num > 0: + payload["logprobs"] = args.top_logprobs_num + + # Merge in extra parameters - these will override defaults if present + payload.update(request_func_input.extra_request_body) + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + if request_func_input.image_data: + payload.update({"image_data": request_func_input.image_data}) + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append( + data["choices"][0]["text"] + ) + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_chat_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + """Makes a request to the OpenAI Chat Completions API. + + Handles both streaming and non-streaming responses, including support + for image data in messages. Calculates and returns various performance + metrics. + + Args: + request_func_input: Input parameters for the request. + pbar: Optional tqdm progress bar to update. + + Returns: + RequestFuncOutput: Output of the request, including generated text, + latency, TTFT, ITL, and success status. + """ + api_url = request_func_input.api_url + assert api_url.endswith( + "chat/completions" + ), "OpenAI Chat Completions API URL must end with 'chat/completions'." + + # TODO put it to other functions when `pbar` logic is refactored + if getattr(args, "print_requests", False): + rid = str(uuid.uuid4()) + input_partial = deepcopy(request_func_input) + input_partial.prompt = "..." + request_start_time = time.time() + print( + f'rid={rid} time={request_start_time} message="request start" request_func_input="{str(input_partial)}"' + ) + + if isinstance(request_func_input.prompt, list): + messages = request_func_input.prompt + elif request_func_input.image_data: + # Build multi-image content: a list of image_url entries followed by the text + content_items = [ + { + "type": "image_url", + "image_url": {"url": img_url}, + } + for img_url in request_func_input.image_data + ] + content_items.append({"type": "text", "text": request_func_input.prompt}) + messages = [ + { + "role": "user", + "content": content_items, + }, + ] + else: + messages = [{"role": "user", "content": request_func_input.prompt}] + + async with _create_bench_client_session() as session: + # Build payload with defaults that can be overridden by extra_request_body + payload = { + "model": request_func_input.model, + "messages": messages, + "max_completion_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + } + + # Add temperature default only if not specified in extra_request_body + if "temperature" not in request_func_input.extra_request_body: + payload["temperature"] = 0.0 + + # Add ignore_eos default only if not specified in extra_request_body + # Default to False for more realistic behavior (respect EOS tokens) + if "ignore_eos" not in request_func_input.extra_request_body: + payload["ignore_eos"] = not args.disable_ignore_eos + + # Merge in extra parameters (tools, temperature, top_p, etc.) + # These will override defaults if present + payload.update(request_func_input.extra_request_body) + + # hack to accommodate different LoRA conventions between SGLang and vLLM. + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + if args.disable_stream: + # Non-streaming response + response_json = await response.json() + output.generated_text = response_json["choices"][0]["message"][ + "content" + ] + output.success = True + output.latency = time.perf_counter() - st + output.ttft = ( + output.latency + ) # For non-streaming, TTFT = total latency + output.output_len = response_json.get("usage", {}).get( + "completion_tokens", output_len + ) + else: + # Streaming response + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # Check if this chunk contains content + delta = data.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + + if content: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.text_chunks.append(content) + output.itl.append( + timestamp - most_recent_timestamp + ) + + most_recent_timestamp = timestamp + generated_text += content + + # Check for usage info in final chunk + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + # TODO put it to other functions when `pbar` logic is refactored + if getattr(args, "print_requests", False): + curr_t = time.time() + output_partial = deepcopy(output) + output_partial.generated_text = "..." + print( + f'rid={rid} time={curr_t} time_delta={curr_t - request_start_time} message="request end" output="{str(output_partial)}"' + ) + + if pbar: + pbar.update(1) + return output + + +async def async_request_truss( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_request_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_sglang_generate( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + ("text" if isinstance(prompt, str) else "input_ids"): prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": request_func_input.output_len, + "ignore_eos": not args.disable_ignore_eos, + }, + "stream": not args.disable_stream, + "lora_path": request_func_input.lora_name, + "return_logprob": args.return_logprob, + "return_routed_experts": args.return_routed_experts, + "logprob_start_len": args.logprob_start_len, + **request_func_input.extra_request_body, + } + if args.top_logprobs_num > 0: + payload["top_logprobs_num"] = args.top_logprobs_num + if args.token_ids_logprob is not None: + payload["token_ids_logprob"] = args.token_ids_logprob + + # Add image data if available (list of image urls/base64) + if request_func_input.image_data: + payload["image_data"] = request_func_input.image_data + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + last_output_len = 0 + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if "text" in data and data["text"]: + timestamp = time.perf_counter() + generated_text = data["text"] + output_len = data["meta_info"]["completion_tokens"] + + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + num_new_tokens = output_len - last_output_len + if num_new_tokens == 0: + continue + chunk_gap = timestamp - most_recent_timestamp + adjust_itl = chunk_gap / num_new_tokens + output.itl.extend([adjust_itl] * num_new_tokens) + + most_recent_timestamp = timestamp + last_output_len = output_len + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + print(f"{output.error=}") + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_embeddings( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + async with _create_bench_client_session() as session: + payload = { + "input": request_func_input.prompt, + "model": request_func_input.model, + } + + if request_func_input.lora_name: + payload["model"] = request_func_input.lora_name + payload["lora_path"] = request_func_input.lora_name + + payload.update(request_func_input.extra_request_body) + + headers = get_request_headers() + if request_func_input.routing_key: + headers[_ROUTING_KEY_HEADER] = request_func_input.routing_key + + output = RequestFuncOutput.init_new(request_func_input) + + st = time.perf_counter() + output.start_time = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + await response.json() + output.latency = time.perf_counter() - st + output.success = True + output.output_len = 0 + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_gserver( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + raise NotImplementedError() + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with _create_bench_client_session() as session: + output = RequestFuncOutput() + try: + if api_url.endswith("/start_profile"): + num_steps = getattr(args, "profile_num_steps", None) + profile_by_stage = getattr(args, "profile_by_stage", None) + if profile_by_stage and num_steps is None: + num_steps = 5 + + output_dir = getattr(args, "profile_output_dir", None) + if output_dir is None: + output_dir = os.getenv("SGLANG_TORCH_PROFILER_DIR", "/tmp") + output_dir = Path(os.path.abspath(os.path.normpath(output_dir))) / str( + time.time() + ) + output_dir.mkdir(exist_ok=True, parents=True) + output_dir = str(output_dir) + + body = { + "activities": getattr(args, "profile_activities", []), + "num_steps": num_steps, + "profile_by_stage": profile_by_stage, + "profile_stages": getattr(args, "profile_stages", None), + "output_dir": output_dir, + "profile_prefix": getattr(args, "profile_prefix", None), + } + else: + # stop_profile doesn't need any parameters + body = {} + print(f"async_request_profile {api_url=} {body=}") + # Add optional profiling parameters if provided + if ( + hasattr(args, "profile_start_step") + and args.profile_start_step is not None + ): + body["start_step"] = str(args.profile_start_step) + if hasattr(args, "profile_steps") and args.profile_steps is not None: + body["num_steps"] = str(args.profile_steps) + async with session.post(url=api_url, json=body) as response: + if response.status == 200: + output.success = True + else: + output.error = ( + (response.reason or "") + ": " + (await response.text()) + ) + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def _build_profile_urls( + profile_prefill_url: Optional[List[str]], + profile_decode_url: Optional[List[str]], +) -> List[Tuple[str, str]]: + """Build profile URLs list from prefill/decode URL arguments. + + Returns: + List of (worker_type, url) tuples. e.g., [("Prefill-0", "http://..."), ("Decode-0", "http://...")] + """ + profile_urls = [] + if profile_prefill_url: + for idx, url in enumerate(profile_prefill_url): + profile_urls.append((f"Prefill-{idx}", url)) + if profile_decode_url: + for idx, url in enumerate(profile_decode_url): + profile_urls.append((f"Decode-{idx}", url)) + return profile_urls + + +async def _call_profile_pd(profile_urls: List[Tuple[str, str]], mode: str) -> None: + """Call profile endpoint (start/stop) on PD separated workers. + + Args: + profile_urls: List of (worker_type, url) tuples + mode: "start" or "stop" + """ + endpoint = "/start_profile" if mode == "start" else "/stop_profile" + action = "Starting" if mode == "start" else "Stopping" + action_past = "started" if mode == "start" else "stopped" + + print(f"{action} profiler...") + + for worker_type, url in profile_urls: + profile_output = await async_request_profile(api_url=url + endpoint) + if profile_output.success: + print(f"Profiler {action_past} for {worker_type} worker at {url}") + else: + print( + f"Failed to {mode} profiler for {worker_type} worker at {url}: {profile_output.error}" + ) + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_sglang_generate, + "sglang-native": async_request_sglang_generate, + "sglang-oai": async_request_openai_completions, + "sglang-oai-chat": async_request_openai_chat_completions, + "sglang-embedding": async_request_openai_embeddings, + "vllm": async_request_openai_completions, + "vllm-chat": async_request_openai_chat_completions, + "lmdeploy": async_request_openai_completions, + "lmdeploy-chat": async_request_openai_chat_completions, + "trt": async_request_trt_llm, + "gserver": async_request_gserver, + "truss": async_request_truss, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_input_text: int + total_input_vision: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p95_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p95_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p95_itl_ms: float + p99_itl_ms: float + max_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p90_e2e_latency_ms: float + p95_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + max_output_tokens_per_s: float = 0.0 + max_concurrent_requests: int = 0 + + +async def get_request( + input_requests: List[DatasetRow], + request_rate: float, + use_trace_timestamps: bool = False, + slowdown_factor: float = 1.0, +) -> AsyncGenerator[DatasetRow, None]: + if use_trace_timestamps: + print( + f"Using trace timestamps for request generation with slowdown factor {slowdown_factor}." + ) + # Sort requests by timestamp for correct replay + input_requests.sort(key=lambda r: r.timestamp) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0].timestamp if input_requests else 0 + + for request in input_requests: + trace_time_s = (request.timestamp - trace_start_time_ms) / 1000.0 + target_arrival_time = start_time + (trace_time_s * slowdown_factor) + + sleep_duration = target_arrival_time - time.perf_counter() + if sleep_duration > 0: + await asyncio.sleep(sleep_duration) + + yield request + else: + input_requests_iter = iter(input_requests) + for request in input_requests_iter: + yield request + + if request_rate == float("inf"): + # If the request rate is infinity, then we don't need to wait. + continue + + # Sample the request interval from the exponential distribution. + interval = np.random.exponential(1.0 / request_rate) + # The next request will be sent after the interval. + await asyncio.sleep(interval) + + +def calculate_metrics( + input_requests: Optional[List[DatasetRow]], + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, + accept_length: Optional[float] = None, + plot_throughput: bool = False, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + total_input_text = 0 + total_input_vision = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + retokenized_itls: List[float] = [] + + use_retokenized_itl = ( + accept_length is not None + and accept_length > 0 + and backend in ("sglang-oai", "sglang-oai-chat") + ) + + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_len + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode(outputs[i].generated_text, add_special_tokens=False) + ) + retokenized_output_lens.append(retokenized_output_len) + if input_requests is not None: + total_input += input_requests[i].prompt_len + total_input_text += input_requests[i].text_prompt_len + total_input_vision += input_requests[i].vision_prompt_len + if output_len > 1: + tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) + if use_retokenized_itl: + for k, itl in enumerate(outputs[i].itl): + num_tokens = len( + tokenizer.encode( + outputs[i].text_chunks[k], add_special_tokens=False + ) + ) + adjusted_itl = itl / num_tokens + retokenized_itls.extend([adjusted_itl] * num_tokens) + else: + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + + e2e_latencies.append(outputs[i].latency) + + completed += 1 + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + + max_output_tokens_per_s = 0.0 + max_concurrent_requests = 0 + + successful_outputs = [output for output in outputs if output.success] + if successful_outputs: + min_start_time = min(output.start_time for output in successful_outputs) + max_end_time = max( + output.start_time + output.latency for output in successful_outputs + ) + + duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1 + tokens_per_second = np.zeros(duration_seconds) + concurrent_requests_per_second = np.zeros(duration_seconds) + + for output in outputs: + if not output.success: + continue + + token_times = [output.start_time + output.ttft] + current_time = token_times[0] + for itl_value in output.itl: + current_time += itl_value + token_times.append(current_time) + + for token_time in token_times: + second_bucket = int(token_time - min_start_time) + if 0 <= second_bucket < duration_seconds: + tokens_per_second[second_bucket] += 1 + + request_start_second = int(output.start_time - min_start_time) + request_end_second = int( + (output.start_time + output.latency) - min_start_time + ) + + for second in range( + request_start_second, min(request_end_second + 1, duration_seconds) + ): + concurrent_requests_per_second[second] += 1 + + if len(tokens_per_second) > 0: + max_output_tokens_per_s = float(np.max(tokens_per_second)) + max_concurrent_requests = int(np.max(concurrent_requests_per_second)) + + if plot_throughput: + if TERM_PLOTLIB_AVAILABLE: + import termplotlib as tpl + + fig = tpl.figure() + fig.plot( + np.arange(len(tokens_per_second)), + tokens_per_second, + title="Output tokens per second", + xlabel="Time (s)", + ) + fig.plot( + np.arange(len(concurrent_requests_per_second)), + concurrent_requests_per_second, + title="Concurrent requests per second", + xlabel="Time (s)", + ) + fig.show() + else: + print("tip: install termplotlib and gnuplot to plot the metrics") + + itls = retokenized_itls if use_retokenized_itl else itls + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_input_text=total_input_text, + total_input_vision=total_input_vision, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p95_ttft_ms=np.percentile(ttfts or 0, 95) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p95_tpot_ms=np.percentile(tpots or 0, 95) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p95_itl_ms=np.percentile(itls or 0, 95) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + max_itl_ms=np.max(itls or 0) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p90_e2e_latency_ms=np.percentile(e2e_latencies, 90) * 1000, + p95_e2e_latency_ms=np.percentile(e2e_latencies, 95) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + max_output_tokens_per_s=max_output_tokens_per_s, + max_concurrent_requests=max_concurrent_requests, + ) + + return metrics, output_lens + + +MULTI_TURN_BACKENDS = {"sglang-oai-chat", "vllm-chat", "lmdeploy-chat"} + + +def wrap_multi_turn_request_func(request_func: Callable, backend: str) -> Callable: + assert ( + backend in MULTI_TURN_BACKENDS + ), f"Multi-turn only supports chat backends: {MULTI_TURN_BACKENDS}, got {backend}" + + async def f( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, + ) -> List[RequestFuncOutput]: + prompts: List[str] = request_func_input.prompt + prev_messages: List[Dict[str, str]] = [] + outputs = [] + + for round_index in range(len(prompts)): + prev_messages.append({"role": "user", "content": prompts[round_index]}) + + inner_input = replace( + copy.deepcopy(request_func_input), prompt=copy.deepcopy(prev_messages) + ) + output = await request_func( + inner_input, pbar=pbar if round_index == len(prompts) - 1 else None + ) + outputs.append(output) + + prev_messages.append( + {"role": "assistant", "content": output.generated_text} + ) + + return outputs + + return f + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[DatasetRow], + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_names: List[str], + lora_request_distribution: Optional[str], + lora_zipf_alpha: Optional[float], + extra_request_body: Dict[str, Any], + profile: bool, + pd_separated: bool = False, + flush_cache: bool = False, + warmup_requests: int = 1, + use_trace_timestamps: bool = False, + mooncake_slowdown_factor=1.0, + mooncake_num_rounds=1, + profile_prefill_url: Optional[List[str]] = None, + profile_decode_url: Optional[List[str]] = None, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Check for multi-turn: prompt is a list of strings (not OpenAI messages dicts) + # Multi-turn format: ["turn1", "turn2", ...] - list of strings + # OpenAI format: [{"role": "user", "content": "..."}, ...] - list of dicts + first_prompt = input_requests[0].prompt + is_multi_turn = ( + isinstance(first_prompt, list) + and len(first_prompt) > 0 + and isinstance(first_prompt[0], str) + ) + if is_multi_turn: + request_func = wrap_multi_turn_request_func(request_func, backend=backend) + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, pbar): + if semaphore is None: + return await request_func(request_func_input=request_func_input, pbar=pbar) + async with semaphore: + return await request_func(request_func_input=request_func_input, pbar=pbar) + + # Warmup + print(f"Starting warmup with {warmup_requests} sequences...") + + # Handle the data structure difference for the warmup request + if args.dataset_name == "mooncake": + # For mooncake, input_requests is a list of dicts. + # We need to build a temporary DatasetRow for the warmup phase. + warmup_record = input_requests[0] + + # Build prompt from hash_ids, just like in the async generator + hash_ids = warmup_record.get("hash_ids", []) + prompt_text = "" + for hash_id in hash_ids: + prompt_text += f"{hash_id}" + " ".join(["hi"] * 512) + prompt_text += "Can you tell me a detailed story in 1000 words?" + + output_len = warmup_record.get("output_length", 32) + prompt_len = len(tokenizer.encode(prompt_text)) + + # Create a temporary DatasetRow object for warmup + test_request = DatasetRow( + prompt=prompt_text, + prompt_len=prompt_len, + output_len=output_len, + image_data=None, # Mooncake doesn't have image data + ) + else: + # For all other datasets, input_requests is a list of DatasetRow objects + test_request = input_requests[0] + + if lora_names is not None and len(lora_names) != 0: + lora_name = lora_names[0] + else: + lora_name = None + + # Create the test input once + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=min(test_request.output_len, 32), + lora_name=lora_name, + image_data=test_request.image_data, + extra_request_body=extra_request_body, + ) + + # Run warmup requests + warmup_tasks = [] + for _ in range(warmup_requests): + warmup_tasks.append( + asyncio.create_task(request_func(request_func_input=test_input)) + ) + + warmup_outputs = await asyncio.gather(*warmup_tasks) + if is_multi_turn: + warmup_outputs = [x for output in warmup_outputs for x in output] + + # Check if at least one warmup request succeeded + if warmup_requests > 0 and not any(output.success for output in warmup_outputs): + raise ValueError( + "Warmup failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {warmup_outputs[0].error}" + ) + else: + print( + f"Warmup completed with {args.warmup_requests} sequences. Starting main benchmark run..." + ) + + # Flush cache + if ("sglang" in backend and _get_bool_env_var("SGLANG_IS_IN_CI")) or flush_cache: + requests.post(base_url + "/flush_cache", headers=get_auth_headers()) + + time.sleep(1.0) + + # Build profile URLs for PD separated mode (do this once at the beginning) + pd_profile_urls = [] + if profile and pd_separated: + pd_profile_urls = _build_profile_urls(profile_prefill_url, profile_decode_url) + if not pd_profile_urls: + print( + "Warning: PD separated mode requires --profile-prefill-url or --profile-decode-url" + ) + print("Skipping profiler start. Please specify worker URLs for profiling.") + + # Start profiler + if profile: + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "start") + else: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + # Run all requests + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + pbar_total = len(input_requests) + if ( + backend == "sglang" and args.dataset_name == "mooncake" + ): # Assuming mooncake is mainly for sglang or similar backends + print("Using time-based Mooncake request scheduler, ignoring --request-rate.") + request_generator = get_mooncake_request_over_time( + input_requests, tokenizer, mooncake_slowdown_factor, mooncake_num_rounds + ) + print( + f"Starting Mooncake trace replay. Sessions: {len(input_requests)}, Rounds per session: {mooncake_num_rounds}. Slowdown factor: {mooncake_slowdown_factor}" + ) + pbar_total *= args.mooncake_num_rounds + else: + request_generator = get_request(input_requests, request_rate) + + # Prepare LoRA request distribution parameters + if lora_request_distribution == "distinct": + lora_idx = 0 + elif lora_request_distribution == "skewed": + weights = np.array([lora_zipf_alpha**-i for i in range(len(lora_names))]) + lora_probs = weights / np.sum(weights) + else: + lora_idx = None + lora_probs = None + + pbar = None if disable_tqdm else tqdm(total=pbar_total) + async for request in request_generator: + if lora_names is not None and len(lora_names) != 0: + if lora_request_distribution == "uniform": + lora_name = random.choice(lora_names) + elif lora_request_distribution == "distinct": + lora_name = lora_names[lora_idx] + lora_idx = (lora_idx + 1) % len(lora_names) + else: + assert ( + lora_request_distribution == "skewed" + ), f"Unexpected lora_request_distribution: {lora_request_distribution}. Expected 'skewed'." + + lora_name = np.random.choice(lora_names, p=lora_probs) + else: + lora_name = None + + # Merge global extra_request_body with per-request extras + # Per-request parameters take precedence over global ones + merged_extra_body = {**extra_request_body, **request.extra_request_body} + + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name=lora_name, + image_data=request.image_data, + extra_request_body=merged_extra_body, + timestamp=request.timestamp, + routing_key=request.routing_key, + ) + + tasks.append( + asyncio.create_task( + limited_request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + if is_multi_turn: + outputs = [x for output in outputs for x in output] + + # Stop profiler (only if profile_steps was not provided, as it auto-stops) + if profile and not ( + hasattr(args, "profile_steps") and args.profile_steps is not None + ): + if pd_separated: + if pd_profile_urls: + await _call_profile_pd(pd_profile_urls, "stop") + else: + if getattr(args, "profile_num_steps", None) is None: + print("Stopping profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/stop_profile" + ) + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + if "sglang" in backend: + server_info = requests.get( + base_url + "/server_info", headers=get_auth_headers() + ) + if server_info.status_code == 200: + server_info_json = server_info.json() + if "decode" in server_info_json: + server_info_json = server_info_json["decode"][0] + if ( + "internal_states" in server_info_json + and server_info_json["internal_states"] + ): + accept_length = server_info_json["internal_states"][0].get( + "avg_spec_accept_length", None + ) + else: + accept_length = None + else: + accept_length = None + else: + accept_length = None + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + input_requests=None if is_multi_turn else input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + accept_length=accept_length, + plot_throughput=args.plot_throughput, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print( + "{:<40} {:<10}".format( + "Traffic request rate:", "trace" if use_trace_timestamps else request_rate + ) + ) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total input text tokens:", metrics.total_input_text)) + if args.dataset_name in ["image", "mmmu"]: + print( + "{:<40} {:<10}".format( + "Total input vision tokens:", metrics.total_input_vision + ) + ) + is_embedding = backend == "sglang-embedding" + if not is_embedding: + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", + metrics.total_output_retokenized, + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + if not is_embedding: + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Peak output token throughput (tok/s):", + metrics.max_output_tokens_per_s, + ) + ) + print( + "{:<40} {:<10}".format( + "Peak concurrent requests:", metrics.max_concurrent_requests + ) + ) + if not is_embedding: + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + if accept_length: + print("{:<40} {:<10.2f}".format("Accept length:", accept_length)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print( + "{:<40} {:<10.2f}".format("P90 E2E Latency (ms):", metrics.p90_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format("P95 E2E Latency (ms):", metrics.p95_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format("P99 E2E Latency (ms):", metrics.p99_e2e_latency_ms) + ) + if not is_embedding: + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P95 TTFT (ms):", metrics.p95_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format( + s="Time per Output Token (excl. 1st token)", n=50, c="-" + ) + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P95 TPOT (ms):", metrics.p95_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-Token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P95 ITL (ms):", metrics.p95_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) + print("=" * 50) + + resp = requests.get(base_url + "/server_info", headers=get_auth_headers()) + server_info = resp.json() if resp.status_code == 200 else None + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "tag": getattr(args, "tag", None), + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": "trace" if use_trace_timestamps else request_rate, + "max_concurrency": max_concurrency, + "sharegpt_output_len": args.sharegpt_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Information + "server_info": server_info, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_input_text_tokens": metrics.total_input_text, + "total_input_vision_tokens": metrics.total_input_vision, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "total_throughput": metrics.total_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p90_e2e_latency_ms": metrics.p90_e2e_latency_ms, + "p95_e2e_latency_ms": metrics.p95_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p95_ttft_ms": metrics.p95_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p95_tpot_ms": metrics.p95_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p95_itl_ms": metrics.p95_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "accept_length": accept_length, + "max_output_tokens_per_s": metrics.max_output_tokens_per_s, + "max_concurrent_requests": metrics.max_concurrent_requests, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "image": + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_" + f"{args.random_output_len}_{args.image_count}imgs_" + f"{args.image_resolution}.jsonl" + ) + elif args.dataset_name.startswith("random"): + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + result_details = { + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + } + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + if args.output_details: + result_for_dump = result | result_details + else: + result_for_dump = result + file.write(json.dumps(result_for_dump) + "\n") + + return result | result_details + + +def check_chat_template(model_path): + try: + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + return "chat_template" in tokenizer.init_kwargs + except Exception as e: + print(f"Fail to load tokenizer config with error={e}") + return False + + +def set_global_args(args_: argparse.Namespace): + """Set the global args.""" + global args + args = args_ + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set default value for warmup_requests if not present + if not hasattr(args, "warmup_requests"): + args.warmup_requests = 1 + + if not hasattr(args, "output_details"): + args.output_details = False + + if not hasattr(args, "tokenize_prompt"): + args.tokenize_prompt = False + + if not hasattr(args, "plot_throughput"): + args.plot_throughput = False + + if not hasattr(args, "top_logprobs_num"): + args.top_logprobs_num = 0 + if not hasattr(args, "token_ids_logprob"): + args.token_ids_logprob = None + if not hasattr(args, "logprob_start_len"): + args.logprob_start_len = -1 + if not hasattr(args, "return_logprob"): + args.return_logprob = False + + if not hasattr(args, "use_trace_timestamps"): + args.use_trace_timestamps = False + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_num_rounds"): + args.mooncake_num_rounds = 1 + + if not hasattr(args, "served_model_name"): + args.served_model_name = None + + if getattr(args, "print_requests", False): + assert args.backend == "sglang-oai-chat" # only support this now + + print(f"benchmark_args={args}") + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + if args.tokenize_prompt: + assert ( + args.backend == "sglang" + ), "`--tokenize-prompt` only compatible with `--backend sglang` currently" + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "sglang-native": 30000, + "sglang-oai": 30000, + "lmdeploy": 23333, + "vllm": 8000, + "trt": 8000, + "gserver": 9988, + "truss": 8080, + }.get(args.backend, 30000) + + # Build base URL with proper IPv6 bracket wrapping (only when base_url is not provided) + if not args.base_url: + _na = NetworkAddress(args.host, args.port) + _host_base = _na.to_url() + else: + _na = None + _host_base = None + + model_url = ( + f"{args.base_url}/v1/models" if args.base_url else f"{_host_base}/v1/models" + ) + + if args.backend == "sglang-embedding": + api_url = ( + f"{args.base_url}/v1/embeddings" + if args.base_url + else f"http://{args.host}:{args.port}/v1/embeddings" + ) + elif args.backend in ["sglang", "sglang-native"]: + api_url = ( + f"{args.base_url}/generate" if args.base_url else f"{_host_base}/generate" + ) + elif args.backend in ["sglang-oai", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/completions" + if args.base_url + else f"{_host_base}/v1/completions" + ) + elif args.backend in ["sglang-oai-chat", "vllm-chat", "lmdeploy-chat"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"{_host_base}/v1/chat/completions" + ) + elif args.backend == "trt": + api_url = ( + f"{args.base_url}/v2/models/ensemble/generate_stream" + if args.base_url + else f"{_host_base}/v2/models/ensemble/generate_stream" + ) + if args.model is None: + print("Please provide a model using `--model` when using `trt` backend.") + sys.exit(1) + elif args.backend == "gserver": + api_url = args.base_url if args.base_url else _na.to_host_port_str() + args.model = args.model or "default" + elif args.backend == "truss": + api_url = ( + f"{args.base_url}/v1/models/model:predict" + if args.base_url + else f"{_host_base}/v1/models/model:predict" + ) + base_url = _host_base if args.base_url is None else args.base_url + + # Wait for server to be ready + if args.ready_check_timeout_sec > 0: + health_url = model_url if args.backend not in ("trt", "gserver") else base_url + if not wait_for_endpoint(health_url, args.ready_check_timeout_sec): + print(f"Server at {health_url} is not ready. Exiting.") + sys.exit(1) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url, headers=get_auth_headers()) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + if args.backend != "sglang-embedding" and not check_chat_template(args.model): + print( + "\nWARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.\n" + "Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.\n" + ) + + if ( + args.backend == "sglang-embedding" + and args.dataset_name in _EMBEDDING_UNSUPPORTED_DATASETS + ): + print(f"{args.dataset_name} dataset is unsupported for embeddings benchmark") + sys.exit(1) + + if args.dataset_name in ["image", "mmmu"]: + args.apply_chat_template = True + assert ( + not args.tokenize_prompt + ), "`--tokenize-prompt` not compatible with image dataset" + + if args.lora_request_distribution in ["distinct", "skewed"]: + assert ( + args.lora_name is not None and len(args.lora_name) > 1 + ), "More than 1 LoRA adapter must be specified via --lora-name to use 'distinct' or 'skewed' request distribution." + + assert ( + args.lora_zipf_alpha > 1 + ), f"Got invalid value for --lora-zipf-alpha of {args.lora_zipf_alpha}. It must be greater than 1." + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.served_model_name or args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + tokenizer = get_tokenizer(tokenizer_id) + input_requests = get_dataset(args, tokenizer, model_id) + + # compatible with SimpleNamespace + if not hasattr(args, "flush_cache"): + args.flush_cache = False + + # Prepare LoRA arguments + lora_request_distribution = ( + args.lora_request_distribution if args.lora_name is not None else None + ) + + lora_zipf_alpha = ( + args.lora_zipf_alpha + if args.lora_name is not None and args.lora_request_distribution == "skewed" + else None + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_names=args.lora_name, + lora_request_distribution=lora_request_distribution, + lora_zipf_alpha=lora_zipf_alpha, + extra_request_body=extra_request_body, + profile=args.profile, + pd_separated=args.pd_separated, + flush_cache=args.flush_cache, + warmup_requests=args.warmup_requests, + use_trace_timestamps=args.use_trace_timestamps, + mooncake_slowdown_factor=args.mooncake_slowdown_factor, + mooncake_num_rounds=args.mooncake_num_rounds, + profile_prefill_url=getattr(args, "profile_prefill_url", None), + profile_decode_url=getattr(args, "profile_decode_url", None), + ) + ) + + +class LoRAPathAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, []) + for lora_name in values: + getattr(namespace, self.dest).append(lora_name) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--ready-check-timeout-sec", + type=int, + default=60, + help="Maximum time in seconds to wait for the server to be ready before benchmarking. Set to 0 to skip. Default: 60.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "custom", + "openai", + "random", + "random-ids", + "generated-shared-prefix", + "mmmu", + "image", + "mooncake", + "longbench_v2", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--served-model-name", + type=str, + help="The name of the model as served by the serving service. If not set, this defaults to the value of --model.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random and image dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random and image dataset.", + ) + # image dataset args + parser.add_argument( + "--image-count", + type=int, + default=1, + help="Number of images per request (only available with the image dataset)", + ) + parser.add_argument( + "--image-resolution", + type=str, + default="1080p", + help=( + "Resolution of images for image dataset. " + "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)." + ), + ) + parser.add_argument( + "--random-image-count", + action="store_true", + help="Enable Random Image Count", + ) + parser.add_argument( + "--image-format", + type=str, + default="jpeg", + help=("Format of images for image dataset. " "Supports jpeg and png."), + ) + parser.add_argument( + "--image-content", + type=str, + default="random", + help=("Content for images for image dataset. " "Supports random and blank."), + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--use-trace-timestamps", + action="store_true", + help="Use timestamps from the trace file for request scheduling. Only valid for 'mooncake' dataset.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--output-details", action="store_true", help="Output details of benchmarking." + ) + parser.add_argument( + "--print-requests", + action="store_true", + help="Print requests immediately during benchmarking. Useful to quickly realize issues.", + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument( + "--top-logprobs-num", + type=int, + default=0, + help="Number of top logprobs to return per token. Only used with --return-logprob.", + ) + parser.add_argument( + "--token-ids-logprob", + type=int, + nargs="+", + default=None, + help="Token IDs to probe logprobs for. E.g. --token-ids-logprob 1 2 10 100 1000. Only used with --return-logprob.", + ) + parser.add_argument( + "--logprob-start-len", + type=int, + default=-1, + help="Start position for returning input logprobs. -1 means no input logprobs, 0 means all. Only used with --return-logprob.", + ) + parser.add_argument( + "--return-routed-experts", + action="store_true", + help="Return routed experts.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--plot-throughput", + action="store_true", + help="Plot throughput and concurrent requests over time. Requires termplotlib and gnuplot.", + ) + # TODO unify all these + parser.add_argument( + "--profile-activities", + type=str, + nargs="+", + default=["CPU", "GPU"], + choices=["CPU", "GPU", "CUDA_PROFILER", "XPU"], + help="Profiler activities to capture: CPU, GPU, XPU, CUDA_PROFILER.", + ) + parser.add_argument( + "--profile-start-step", + type=int, + default=None, + help="Start profiling after this many forward steps. Useful for warmup.", + ) + parser.add_argument( + "--profile-steps", + type=int, + default=None, + help="Number of steps to profile. If specified, profiling stops automatically after this many steps.", + ) + parser.add_argument("--profile-num-steps", type=int, default=None) + parser.add_argument("--profile-by-stage", action="store_true", default=False) + parser.add_argument("--profile-stages", nargs="+", default=None) + parser.add_argument( + "--profile-output-dir", + type=str, + default=None, + help="Output directory for profile traces.", + ) + parser.add_argument( + "--profile-prefix", + type=str, + default=None, + help="Prefix for profile trace filenames.", + ) + parser.add_argument( + "--lora-name", + type=str, + nargs="*", + default=None, + action=LoRAPathAction, + help="The names of LoRA adapters. You can provide a list of names in the format {name} {name} {name}...", + ) + parser.add_argument( + "--lora-request-distribution", + type=str, + default="uniform", + choices=[ + "uniform", + "distinct", + "skewed", + ], + help="What distribution to sample the LoRA adapters specified in --lora-name. Borrowed from the Punica paper. " + "'distinct' distribution means selecting a new LoRA adapter for every request. " + "'skewed' distribution follows the Zipf distribution, where the number of requests " + "to model i specified in --lora-name is α times the number of requests for model i+1, " + "where α > 1.", + ) + parser.add_argument( + "--lora-zipf-alpha", + type=float, + default=1.5, + help="The parameter to use for the Zipf distribution when --lora-request-distribution='skewed'.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--pd-separated", + action="store_true", + help="Benchmark PD disaggregation server", + ) + + # Create a mutually exclusive group for profiling URLs + # In PD separated mode, prefill and decode workers must be profiled separately + profile_url_group = parser.add_mutually_exclusive_group() + profile_url_group.add_argument( + "--profile-prefill-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the prefill worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-prefill-url http://localhost:30000 http://localhost:30001. " + "NOTE: Cannot be used together with --profile-decode-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + profile_url_group.add_argument( + "--profile-decode-url", + type=str, + nargs="*", + default=None, + help="URL(s) of the decode worker(s) for profiling in PD separated mode. " + "Can specify multiple URLs: --profile-decode-url http://localhost:30010 http://localhost:30011. " + "NOTE: Cannot be used together with --profile-prefill-url. " + "In PD separated mode, prefill and decode workers must be profiled separately.", + ) + parser.add_argument( + "--flush-cache", + action="store_true", + help="Flush the cache before running the benchmark", + ) + parser.add_argument( + "--warmup-requests", + type=int, + default=1, + help="Number of warmup requests to run before the benchmark", + ) + parser.add_argument( + "--tokenize-prompt", + action="store_true", + help="Use integer ids instead of string for inputs. Useful to control prompt lengths accurately", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + parser.add_argument( + "--gsp-range-ratio", + type=float, + # WARN: The default 1.0 is for backward compatibility, and is different from the default 0.0 for random dataset + default=1.0, + help="Range of sampled ratio of input/output length, used only for gsp dataset.", + ) + group.add_argument( + "--gsp-fast-prepare", + action="store_true", + help="Speedup preparing by removing statistics computation, which will make some output statistics inaccurate but suitable for pressure tests.", + ) + group.add_argument( + "--gsp-send-routing-key", + action="store_true", + help="Send routing key in requests via X-SMG-Routing-Key header. Requests with the same prefix share the same routing key.", + ) + group.add_argument( + "--gsp-num-turns", + type=int, + default=1, + help="Number of turns for multi-turn conversations. If > 1, each prompt becomes a list of questions sharing the same system prefix.", + ) + group.add_argument( + "--gsp-ordered", + action="store_true", + help="Keep requests in order without shuffling. By default, requests are shuffled randomly.", + ) + mooncake_group = parser.add_argument_group("mooncake dataset arguments") + mooncake_group.add_argument( + "--mooncake-slowdown-factor", + type=float, + default=1.0, + help="Slowdown factor for replaying the mooncake trace. " + "A value of 2.0 means the replay is twice as slow. " + "NOTE: --request-rate is IGNORED in mooncake mode.", + ) + mooncake_group.add_argument( + "--mooncake-num-rounds", + type=int, + default=1, + help="Number of conversation rounds for each session in the mooncake dataset. " + "A value > 1 will enable true multi-turn session benchmarking.", + ) + mooncake_group.add_argument( + "--mooncake-workload", + type=str, + default="conversation", + choices=[ + "mooncake", + "conversation", + "synthetic", + "toolagent", + ], + help="Underlying workload for the mooncake dataset.", + ) + parser.add_argument( + "--tag", type=str, default=None, help="The tag to be dumped to output." + ) + parser.add_argument( + "--header", + type=str, + nargs="+", + default=None, + help="Custom HTTP headers in Key=Value format. Example: --header MyHeader=MY_VALUE MyAnotherHeader=myanothervalue", + ) + args = parser.parse_args() + run_benchmark(args) diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh new file mode 100755 index 0000000..4200124 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape +# on Kunlun P800 (DeepSeek-V4-Flash-INT8). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# The shared library's GPU monitor uses nvidia-smi; override it with an +# xpu-smi sampler that emits the same CSV shape. +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + local ep_args="" + if (( dp == 1 )); then + ep_args="--ep-size ${EP_SIZE:-$tp}" + fi + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + if (( dp > 1 )); then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="" + log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" + if [[ "$output_file" == "/dev/null" ]]; then + container_output="/dev/null" + fi + + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --model "$MODEL_PATH" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --output-file "$container_output" + --output-details + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? + + if [[ "$output_file" != "/dev/null" ]]; then + docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1 + fi +} + +export -f engine_run_bench +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE +export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..5118d31 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_adaptive_concurrency_add16.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each SGLang TP/DP/ISL/OSL shape +# on Kunlun P800 (DeepSeek-V4-Flash-INT8). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="sglang" +ENGINE_PORT="$SGLANG_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# The shared library's GPU monitor uses nvidia-smi; override it with an +# xpu-smi sampler that emits the same CSV shape. +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + log "stopping container ${CONTAINER_NAME} tp=${tp} dp=${dp}" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + local ep_args="" + if (( dp == 1 )); then + ep_args="--ep-size ${EP_SIZE:-$tp}" + fi + local args="--host 0.0.0.0 --port ${ENGINE_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + if (( dp > 1 )); then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/sglang_tp${tp}_dp${dp}.server.outer.log" + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: sglang health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="" + log "sglang server healthy tp=${tp} dp=${dp} log=docker logs ${CONTAINER_NAME}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" + if [[ "$output_file" == "/dev/null" ]]; then + container_output="/dev/null" + fi + + local -a bench_args=( + --backend sglang + --host 127.0.0.1 + --port "$ENGINE_PORT" + --model "$MODEL_PATH" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --output-file "$container_output" + --output-details + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$CONTAINER_DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs 2>/dev/null || true + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? + + if [[ "$output_file" != "/dev/null" ]]; then + docker cp "$CONTAINER_NAME:${container_output}" "$output_file" || return 1 + fi +} + +export -f engine_run_bench +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE +export BENCH_DATASET_NAME CONTAINER_DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 +# If the initial concurrency violates the TTFT SLO, search downward. Stop at +# the first acceptable value (16 -> 8; only try 1 when 8 still violates it). +export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" +# When concurrency 1 still has a severely excessive TTFT, stop the remaining +# shapes in this TP/DP group. Zero disables this rule. +export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}" + +adaptive_main "$@" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_bench.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_bench.sh new file mode 100755 index 0000000..819dcc1 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/run_bench.sh @@ -0,0 +1,559 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for DeepSeek-V4-Flash-INT8 on Kunlun P800 SGLang (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +PYTHON="${PYTHON:-$(command -v python3)}" + +# Export variables used inside functions that are called via bash -c subshells. +export CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH SGLANG_PORT CONTAINER_DATASET_PATH + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1 +} + +container_running() { + docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + # The server only ever runs inside this experiment's dedicated container, + # so removing the container is a complete and safe stop. + if container_running; then + log "stopping container ${CONTAINER_NAME} (tp=${tp}, dp=${dp})" + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + else + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + fi + sleep 2 +} + +build_server_args() { + # Keep in sync with the launch args in start_sglang_docker.sh. + local tp="$1" + local dp="$2" + + local ep_args="" + if (( dp == 1 )); then + ep_args="--ep-size ${EP_SIZE:-$tp}" + fi + local args="--host 0.0.0.0 --port ${SGLANG_PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${tp} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + if [[ "$dp" -gt 1 ]]; then + args="${args} --dp-size ${dp}" + fi + printf '%s' "$args" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting sglang server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_sglang_docker.sh" "$tp" "$dp" \ + >> "${log_dir_global}/sglang_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + if ! is_server_healthy; then + log "error: sglang server tp=${tp} dp=${dp} failed health check on port ${SGLANG_PORT}" + return 1 + fi + log "sglang server tp=${tp} dp=${dp} is healthy on port ${SGLANG_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting sglang server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving inside the server container (the P800 image + # carries the only working client environment for this fork). + docker exec "$CONTAINER_NAME" \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + "$CONTAINER_PYTHON" -m sglang.bench_serving "$@" +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --model "'"$MODEL_PATH"'" \ + --dataset-name random \ + --dataset-path "'"$CONTAINER_DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --random-range-ratio 1.0 \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' || log "WARNING: warmup failed; continuing with config" + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" 2>/dev/null; then + return 0 + fi + docker logs --tail 2000 "$CONTAINER_NAME" 2>/dev/null | grep -Eiq "$pattern" +} + +start_gpu_monitor() { + # Sample XPU memory/utilization with xpu-smi and emit nvidia-smi-style CSV + # rows so scripts/common/parse_backend.py can read them unchanged. + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + ( + echo "timestamp, index, memory.used [MiB], memory.total [MiB], utilization.gpu [%]" + while true; do + xpu-smi 2>/dev/null | awk -F'|' -v ts="$(date '+%Y/%m/%d %H:%M:%S')" ' + /^\|[[:space:]]*[0-9]+[[:space:]]/ { + l=$0; sub(/^\|[[:space:]]*/, "", l); sub(/[[:space:]].*$/, "", l); idx=l + } + /MiB/ && $4 ~ /%/ { + split($3, a, "/"); u=a[1]; t=a[2] + gsub(/[^0-9.]/, "", u); gsub(/[^0-9.]/, "", t) + r=$4; gsub(/[^0-9.]/, "", r) + printf "%s, %s, %s MiB, %s MiB, %s %%\n", ts, idx, u, t, r + }' + sleep "$GPU_MEM_SAMPLE_INTERVAL_S" + done + ) >> "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +scenario_config_json() { + "$PYTHON" -c " +import json, sys +print(json.dumps({ + 'phase': 'main', + 'concurrency': int(sys.argv[1]), + 'input_len': int(sys.argv[2]), + 'output_len': int(sys.argv[3]), + 'dataset': 'random', + 'num_prompts': int(sys.argv[4]), +})) +" "$1" "$2" "$3" "$4" +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${osl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "sglang" \ + "sglang-xpu" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/p800/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "P800 SGLang TP×DP matrix for DeepSeek-V4-Flash-INT8" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + "$PYTHON" - "$tp" "$dp" "$server_args_str" "${result_root}/results.json" <<'PY' +import json, os, sys + +tp, dp, server_args, path = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], sys.argv[4] +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) +data["config"] = { + "tp": tp, + "dp": dp, + "xpu_visible_devices": os.environ.get("XPU_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7"), + "backend": "sglang", + "engine": "sglang-xpu", + "server_start_script": "experiments/p800/dsv4_p800_sglang_tp_dp_matrix/start_sglang_docker.sh", + "server_args": server_args, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl osl conc num + local output_file container_output detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${osl}" + output_file="${raw_dir}/sglang_main_${conc}_${isl}_${osl}.jsonl" + container_output="/tmp/bench_outputs/sglang_main_${conc}_${isl}_${osl}.jsonl" + detail_log="${phase_log_dir}/sglang_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + if ! container_running; then + log "WARNING: container ${CONTAINER_NAME} died, skipping remaining scenarios in ${config_label}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_CONTAINER_DIED" "container died" "$tp" "$dp" + break + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + docker exec "$CONTAINER_NAME" mkdir -p /tmp/bench_outputs + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend sglang \ + --host 127.0.0.1 \ + --port "'"$SGLANG_PORT"'" \ + --model "'"$MODEL_PATH"'" \ + --dataset-name random \ + --dataset-path "'"$CONTAINER_DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$osl"'" \ + --random-range-ratio 1.0 \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$container_output"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + if docker cp "${CONTAINER_NAME}:${container_output}" "$output_file"; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + log "ERROR: failed to copy ${container_output} from container" + bench_rc=1 + fi + + # Failure handling. + if detect_oom "$detail_log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"OOM\"" \ + "note=\"detected out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(scenario_config_json "$conc" "$isl" "$osl" "$num")" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=sglang" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh new file mode 100755 index 0000000..c78c399 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_docker.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Start the P800 SGLang INT8 server in Docker for a given TP×DP configuration. +# Usage: start_sglang_docker.sh +# +# Based on the proven P800 INT8 launch from dsv4_p800_long_context_matrix, +# with --tensor-parallel-size/--dp-size parameterized per config. +set -Eeuo pipefail + +TP="${1:-}" +DP="${2:-}" +if [[ -z "$TP" || -z "$DP" ]]; then + echo "Usage: $0 " + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +PORT="${SGLANG_PORT:-30014}" +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" +SERVER_LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_sglang_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" + +log "starting P800 SGLang INT8 server (tp=${TP}, dp=${DP})" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "container: ${CONTAINER_NAME}" + +# Stop any existing container with the same name. +docker rm -f "$CONTAINER_NAME" 2>/dev/null || true + +# Build device args. +device_args="" +for i in 0 1 2 3 4 5 6 7; do + device_args="${device_args} --device /dev/xpu${i}:/dev/xpu${i}" +done +device_args="${device_args} --device /dev/xpuctrl:/dev/xpuctrl" + +# Environment variables required by the P800 SGLang INT8 image. +env_args=( + -e XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES}" + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + -e CUDA_DEVICE_ORDER=OAM_ID + -e SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 + -e XMLIR_FORCE_USE_XPU_GRAPH=1 + -e SGLANG_DSV4_MODE=2604 + -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + -e SGLANG_NSA_DUAL_STREAM=true + -e SGLANG_NSA_QUANT_WQ_B_WK=false + -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 + -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false + -e SGLANG_OPT_USE_TILELANG_MHC_PRE=1 + -e SGLANG_OPT_USE_TILELANG_MHC_POST=1 + -e SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 + -e SGLANG_SET_CPU_AFFINITY=1 + -e SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 + -e XSGL_INTERTYPE_BFP16=1 + -e ENABLE_FAST_BFP16_ATTN=1 + -e XSGL_USE_DEEP_GEMM_BMM=1 + -e XSGL_XDNN_QUANT=1 + -e XSGL_FUSE_RMS_NORM_QUANT=1 + -e XSGL_TRANSPOSE_MATMUL_WEIGHT=1 + -e XINFER_QUANT_SDNN=1 + -e XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 + -e XSGL_EARLY_FIRST_TOKEN=1 + -e XSGL_ENABLE_TGEMM_FP16=1 + -e SGLANG_ENABLE_SPEC_V2=True + -e SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 + -e PYTHONDONTWRITEBYTECODE=1 + -e XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops + -e XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib + -e BKCL_TREE_THRESHOLD=1048576 + -e CUDA_ENABLE_P2P_NO_UVA=1 + -e NCCL_IB_GID_INDEX=3 + -e IS_DSV4=1 + -e MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json + # INT8 specific + -e SGLANG_DSV4_FP4_EXPERTS=false + -e SGLANG_APPLY_CONFIG_BACKUP=auto + -e BKCL_ENABLE_XDR=1 + -e BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 + -e BKCL_RDMA_VERBS=1 + -e XSGL_INT8_LM_HEAD=1 + -e SGLANG_P800_ALL_GATHER_FALLBACK=0 +) + +# Launch args: baseline P800 INT8 command (same as the tp_dp_matrix +# experiment) PLUS the official w8a8 run_server.sh parameters: +# EAGLE speculative decoding (steps/topk/draft from config.env) +# --chunked-prefill-size / --max-prefill-tokens (official long-prefill chunking) +# --max-running-requests 64 (official cap) +# --ep-size is only added when DP=1: this vendor image crashes with +# ZeroDivisionError when EP spans DP replicas (probed 2026-07-17). +ep_args="" +if [[ "$DP" -eq 1 ]]; then + ep_args="--ep-size ${EP_SIZE:-$TP}" +fi +launch_args="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static ${MEM_FRACTION_STATIC} --tensor-parallel-size ${TP} ${ep_args} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length ${CONTEXT_LENGTH} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --max-prefill-tokens ${MAX_PREFILL_TOKENS} --max-running-requests ${MAX_RUNNING_REQUESTS} --speculative-algorithm EAGLE --speculative-num-steps ${SPEC_NUM_STEPS} --speculative-eagle-topk ${SPEC_EAGLE_TOPK} --speculative-num-draft-tokens ${SPEC_NUM_DRAFT_TOKENS}" + +if [[ "$DP" -gt 1 ]]; then + launch_args="${launch_args} --dp-size ${DP}" +fi + +# Base64-encode the bootstrap command to avoid host-shell quoting issues. +server_cmd=$(cat </dev/null || true +/root/miniconda/envs/python310_torch25_cuda/bin/pip install --upgrade safetensors -q +/root/miniconda/envs/python310_torch25_cuda/bin/pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q +/root/miniconda/envs/python310_torch25_cuda/bin/python -c "import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')" ${launch_args} +EOF +) +server_cmd_b64=$(printf '%s' "$server_cmd" | base64 -w0) + +patch_mounts=( + -v "${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro" +) +# Patch the image's sglang.bench_serving to also report TTFT/TPOT/E2E P95. +# The P800 image only ships median/P99 for those metrics, and the adaptive +# search reads p95_ttft_ms for its TTFT SLO stop. The container is recreated +# per TP/DP config, so the patch must be mounted at every `docker run`. +BENCH_SERVING_PATCH="${SCRIPT_DIR}/patches/bench_serving.py" +if [[ -f "${BENCH_SERVING_PATCH}" ]]; then + patch_mounts+=( + -v "${BENCH_SERVING_PATCH}:/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang/bench_serving.py:ro" + ) +fi +# Mount the ShareGPT seed dataset only when it exists on the host, so that +# BENCH_DATASET_NAME=random-ids (no seed needed) never makes docker create a +# stray directory at DATASET_PATH. +if [[ -f "${DATASET_PATH}" ]]; then + patch_mounts+=( -v "${DATASET_PATH}:${CONTAINER_DATASET_PATH}:ro" ) +fi + +echo "=== Starting P800 SGLang INT8 server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: ${DOCKER_IMAGE}" +echo "Model: ${MODEL_PATH}" +echo "Container name: ${CONTAINER_NAME}" +echo "Host port: ${PORT}" +echo "Command: sglang.launch_server ${launch_args}" +echo "Log: docker logs ${CONTAINER_NAME}" + +docker run -d \ + --name "${CONTAINER_NAME}" \ + --privileged \ + --network host \ + --ipc host \ + ${device_args} \ + -v "${MODEL_PATH}:/models:ro" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + "${patch_mounts[@]}" \ + "${env_args[@]}" \ + "${DOCKER_IMAGE}" \ + bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ + >> "${SERVER_LOG}" 2>&1 + +log "container ${CONTAINER_NAME} started, waiting for health" +healthy=0 +for ((i = 1; i <= 600; i++)); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + healthy=1 + break + fi + if [[ "$(docker inspect -f '{{.State.Running}}' "${CONTAINER_NAME}" 2>/dev/null)" != "true" ]]; then + log "ERROR: container ${CONTAINER_NAME} exited during startup; last server logs:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 + fi + sleep 1 +done +if [[ "$healthy" == "1" ]]; then + log "container ${CONTAINER_NAME} is healthy" + exit 0 +else + log "ERROR: container ${CONTAINER_NAME} failed health check; last server logs:" + docker logs --tail 100 "${CONTAINER_NAME}" 2>&1 || true + exit 1 +fi diff --git a/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_dp.sh b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_dp.sh new file mode 100755 index 0000000..8939f73 --- /dev/null +++ b/experiments/p800/dsv4_p800_sglang_tp_dp_official/start_sglang_dp.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Start SGLang server for a given TP×DP configuration. +# Usage: start_sglang_dp.sh +# +# P800 SGLang always runs in the vendor Docker image, so this simply +# delegates to the Docker start script. Kept under the same name as the +# H20 experiment for parity. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${SCRIPT_DIR}/start_sglang_docker.sh" "$@"