qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add master orchestrator script for sequential benchmark execution

Browse Source 
- Implemented a bash script to run six benchmark experiments sequentially.
- Added features for fault tolerance, out-of-memory recovery, and checkpoint management.
- Included detailed logging for monitoring experiment progress and results.
- Integrated GPU memory cleanup checks before each experiment.
- Provided functionality to resume experiments from checkpoints.
This commit is contained in:
Quantong Qiu 2026-07-13 02:46:15 +00:00
parent 302d1a9822
commit 372ff08eea
35 changed files with 13671 additions and 4769 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py
View File

@ -228,16 +228,22 @@ class LoadBalancer:
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
import uuid
def make_prompt(token_len: int) -> str:
prefix = str(uuid.uuid4())
words = [
"the", "quick", "brown", "fox",
"jumps", "over", "lazy", "dog",
"benchmark", "performance", "latency"
]
tokens = [prefix]
while len(tokens) < token_len:
tokens.extend(words)
return " ".join(tokens[:needed])
return " ".join(tokens[:token_len])
# ---------------------------------------------------------------------------
@ -440,6 +446,35 @@ async def run_benchmark(
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
print("Starting warmup...")
warmup_per_service = 10
warmup_requests = warmup_per_service * len(ports)
warmup_prompts = [
make_prompt(input_len)
for _ in range(warmup_requests)
]
warmup_tasks = [
send_request_with_retry(
session=session,
host=host,
lb=lb,
model=model,
request_id=-1,
prompt=p,
max_tokens=output_len,
temperature=temperature,
)
for p in warmup_prompts
]
await asyncio.gather(*warmup_tasks)
print("Warmup finished.")
# 给 CUDA / Scheduler 一点时间稳定
await asyncio.sleep(2)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)

184
experiments/dsv4_h200_vllm_tp2_custom_bench/config.env
View File

@ -34,50 +34,146 @@ LB_STRATEGY="round_robin"
# For multi-service we include low-concurrency scenarios to show how
# requests spread across 4 services.
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 1024 128 20"
"1 1024 256 20"
"1 1024 512 20"
"1 1024 1024 20"
"1 1024 2048 20"
"1 1024 4096 20"
"8 1024 128 160"
"8 1024 256 160"
"8 1024 512 160"
"8 1024 1024 160"
"8 1024 2048 160"
"8 1024 4096 160"
"32 1024 128 640"
"32 1024 256 640"
"32 1024 512 640"
"32 1024 1024 640"
"32 1024 2048 640"
"32 1024 4096 640"
"128 1024 128 2560"
"128 1024 256 2560"
"128 1024 512 2560"
"128 1024 1024 2560"
"128 1024 2048 2560"
"128 1024 4096 2560"
"1 2048 128 20"
"1 2048 256 20"
"1 2048 512 20"
"1 2048 1024 20"
"1 2048 2048 20"
"1 2048 4096 20"
"8 2048 128 160"
"8 2048 256 160"
"8 2048 512 160"
"8 2048 1024 160"
"8 2048 2048 160"
"8 2048 4096 160"
"32 2048 128 640"
"32 2048 256 640"
"32 2048 512 640"
"32 2048 1024 640"
"32 2048 2048 640"
"32 2048 4096 640"
"128 2048 128 2560"
"128 2048 256 2560"
"128 2048 512 2560"
"128 2048 1024 2560"
"128 2048 2048 2560"
"128 2048 4096 2560"
"1 4096 128 20"
"1 4096 256 20"
"1 4096 512 20"
"1 4096 1024 20"
"1 4096 2048 20"
"1 4096 4096 20"
"8 4096 128 160"
"8 4096 256 160"
"8 4096 512 160"
"8 4096 1024 160"
"8 4096 2048 160"
"8 4096 4096 160"
"32 4096 128 640"
"32 4096 256 640"
"32 4096 512 640"
"32 4096 1024 640"
"32 4096 2048 640"
"32 4096 4096 640"
"128 4096 128 2560"
"128 4096 256 2560"
"128 4096 512 2560"
"128 4096 1024 2560"
"128 4096 2048 2560"
"128 4096 4096 2560"
"1 16384 128 20"
"1 16384 256 20"
"1 16384 512 20"
"1 16384 1024 20"
"1 16384 2048 20"
"8 16384 128 160"
"8 16384 256 160"
"8 16384 512 160"
"8 16384 1024 160"
"8 16384 2048 160"
"32 16384 128 640"
"32 16384 256 640"
"32 16384 512 640"
"32 16384 1024 640"
"32 16384 2048 640"
"128 16384 128 2560"
"128 16384 256 2560"
"128 16384 512 2560"
"128 16384 1024 2560"
"128 16384 2048 2560"
"1 65536 128 20"
"1 65536 256 20"
"1 65536 512 20"
"1 65536 1024 20"
"8 65536 128 160"
"8 65536 256 160"
"8 65536 512 160"
"8 65536 1024 160"
"32 65536 1024 160"
"1 131072 128 20"
"1 131072 256 20"
"1 131072 512 20"
"8 131072 512 160"
"1 262144 256 20"
"8 262144 128 160"
"1 524288 128 20"
)
# Server start script bundled with this experiment.

47
experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json
View File

@ -1,47 +0,0 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091522",
"timestamp": "2026-07-10T09:15:22+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},
"config": {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20",
"4 512 256 80",
"8 512 256 160",
"16 512 256 320",
"32 512 256 640",
"64 512 256 1280",
"128 512 256 2560",
"1 2048 512 20",
"8 2048 512 160",
"32 2048 512 640",
"128 2048 512 2560",
"1 4096 1024 20",
"8 4096 1024 160",
"32 4096 1024 640",
"128 4096 1024 2560"
]
},
"scenarios": []
}

47
experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json
View File

@ -1,47 +0,0 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091700",
"timestamp": "2026-07-10T09:17:00+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},
"config": {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20",
"4 512 256 80",
"8 512 256 160",
"16 512 256 320",
"32 512 256 640",
"64 512 256 1280",
"128 512 256 2560",
"1 2048 512 20",
"8 2048 512 160",
"32 2048 512 640",
"128 2048 512 2560",
"1 4096 1024 20",
"8 4096 1024 160",
"32 4096 1024 640",
"128 4096 1024 2560"
]
},
"scenarios": []
}

6
experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json → experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json
View File

@ -1,8 +1,8 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091141",
"timestamp": "2026-07-10T09:11:41+00:00",
"run_id": "20260710-104930",
"timestamp": "2026-07-10T10:49:30+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
@ -11,7 +11,7 @@
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_commit": "def1355",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},

273
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md
View File

@ -1,273 +0,0 @@
# dsv4_h200_vllm_tp2_custom_bench_no_fp8
> **Note:** This variant does **NOT** use `--kv-cache-dtype fp8`.
## 目的
当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU：
| 服务 | 端口 | GPU 对 |
|------|------|--------|
| 1 | 30005 | 0, 1 |
| 2 | 30006 | 2, 3 |
| 3 | 30007 | 4, 5 |
| 4 | 30008 | 6, 7 |
此时 `sglang.bench_serving` 存在以下问题：
1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。
2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。
3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。
因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。
## 与 sglang.bench_serving 的区别
| 特性 | sglang.bench_serving | bench_client.py (本实验) |
|------|----------------------|--------------------------|
| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) |
| 并发控制 | 进程级 | `asyncio.Semaphore` |
| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` |
| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 |
| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 |
| 低并发精度 | 一般 | 高（适合 TP=2 场景） |
| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 |
## 文件说明
| 文件 | 说明 |
|------|------|
| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 |
| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） |
| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 |
| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） |
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
## 快速运行
```bash
# 完整流程（自动启动 4 个服务、压测、生成报告）
bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh
# 跳过服务管理（已有 4 个服务在运行）
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh
# 单独运行压测客户端（多服务，带健康检查）
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \
--host 127.0.0.1 \
--ports 30005,30006,30007,30008 \
--model deepseek-v4-flash \
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
--lb-strategy round_robin \
--health-check-interval 5 \
--health-max-failures 2 \
--health-recovery-interval 10 \
--request-max-retries 2 \
--output-file /tmp/test.jsonl
# 单独运行压测客户端（单服务，向后兼容）
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \
--output-file /tmp/test_single.jsonl
# 解析已有结果
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \
experiments/dsv4_h200_vllm_tp2_custom_bench/results/<RUN_ID>
```
## 场景设计
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|------|----------|----------|--------|------|
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req |
| 8 | 512 | 256 | 160 | 中低并发 |
| 16 | 512 | 256 | 320 | 中等并发 |
| 32 | 512 | 256 | 640 | 中高并发 |
| 64 | 512 | 256 | 1280 | 高并发 |
| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 |
| 1 | 2048 | 512 | 20 | 长输入基线 |
| 8 | 2048 | 512 | 160 | 长输入中并发 |
| 32 | 2048 | 512 | 640 | 长输入高并发 |
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
## 健康检查与故障自动跳过
`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。
### 健康检查机制
| 参数 | 默认值 | 说明 |
|------|--------|------|
| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 |
| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 |
| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 |
| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 |
### 故障处理流程
```
请求 → 负载均衡选择端口 → 发送请求
↓
成功 → 记录结果
↓
失败 → 换另一个端口重试（最多 --request-max-retries 次）
↓
全部失败 → 记录失败，继续下一个请求
```
后台同时运行健康检查：
- 每 5 秒对所有端口执行 `/health` 探测
- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求
- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池
### 输出中的故障信息
汇总 JSON 包含以下字段：
```json
{
"completed": 638,
"failed": 2,
"retried": 5,
"healthy_ports_at_end": [30005, 30006, 30007],
"per_port_stats": {
"30005": {"count": 160, "failed": 0, ...},
"30006": {"count": 159, "failed": 1, ...},
"30007": {"count": 160, "failed": 0, ...},
"30008": {"count": 159, "failed": 1, ...}
}
}
```
- `failed`: 最终失败的请求数（重试后仍失败）
- `retried`: 至少重试过一次的请求数
- `healthy_ports_at_end`: 压测结束时仍健康的服务端口
- `per_port_stats[].failed`: 每个端口的失败数
### 场景间健康检查
`run_bench.sh` 在每个场景结束后会打印一次服务健康状态：
```
[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[]
[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008]
[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008
```
这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。
### 自定义故障参数
```bash
python3 bench_client.py \
--ports 30005,30006,30007,30008 \
--health-check-interval 3 \
--health-max-failures 1 \
--health-recovery-interval 5 \
--request-max-retries 3 \
...
```
- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复
- 更容错：单个请求最多重试 3 次
## 负载均衡策略
`bench_client.py` 支持三种负载均衡策略：
| 策略 | 说明 | 适用场景 |
|------|------|----------|
| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 |
| `random` | 随机选择一个端口 | 简单，分布可能不均 |
| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 |
通过 `--lb-strategy` 参数选择：
```bash
python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ...
```
## 输出格式
### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`)
每行一个 JSON 对象，包含请求分发到的目标端口：
```json
{
"request_id": 0,
"input_tokens": 512,
"output_tokens": 256,
"first_token_time_ms": 45.23,
"e2e_latency_ms": 5234.56,
"inter_token_latencies": [18.5, 19.2, ...],
"tpot_ms": 19.8,
"mean_itl_ms": 19.5,
"success": true,
"error": "",
"target_port": 30005
}
```
最后一行是汇总数据，包含 **各端口统计**：
```json
{
"completed": 640,
"failed": 0,
"duration": 12.34,
"request_throughput": 51.88,
"mean_ttft_ms": 48.5,
"p95_ttft_ms": 62.3,
"per_port_stats": {
"30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0},
"30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0},
"30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0},
"30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0}
},
"lb_strategy": "round_robin",
"num_services": 4
}
```
### 结构化结果 (`results.json`)
遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。
### 人类可读报告 (`report.md`)
Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。
## 单独启动/停止 4 个服务
```bash
# 启动 4 个服务
bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh
# 停止（run_bench.sh 会自动停止，但也可以手动）
for port in 30005 30006 30007 30008; do
pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid"
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
rm -f "$pid_file"
done
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
```
## 注意事项
- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。
- 默认 `temperature=0.0` 以保证输出长度稳定。
- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。
- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。
- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。
- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。
- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。

617
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py
View File

@ -1,617 +0,0 @@
#!/usr/bin/env python3
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
Sends concurrent requests to multiple inference servers and records per-request
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
Designed for TP=2 on 8-GPU setups where 4 independent services run on
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
round-robin or random load balancing. Unhealthy services are automatically
skipped with periodic retry.
Usage:
# Single service (backward compatible)
python3 bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--output-file results.jsonl
# Multi-service (4 services on ports 30005-30008)
python3 bench_client.py \
--host 127.0.0.1 \
--ports 30005,30006,30007,30008 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--lb-strategy round_robin \
--output-file results.jsonl
The output JSONL contains one object per request with raw timing data.
A final summary line ("completed" field) aggregates all requests.
"""
import argparse
import asyncio
import json
import random
import sys
import time
import urllib.request
from dataclasses import asdict, dataclass, field
from typing import Any
# Try aiohttp first; fall back to urllib for stdlib-only environments.
try:
import aiohttp
HAS_AIOHTTP = True
except ImportError:
HAS_AIOHTTP = False
# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------
@dataclass
class RequestResult:
request_id: int
input_tokens: int = 0
output_tokens: int = 0
first_token_time_ms: float = 0.0 # TTFT
e2e_latency_ms: float = 0.0 # total wall time
inter_token_latencies: list[float] = field(default_factory=list)
success: bool = False
error: str = ""
target_port: int = 0 # which service handled this request
retry_count: int = 0 # how many times this request was retried
@property
def tpot_ms(self) -> float:
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
if self.output_tokens <= 1:
return 0.0
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
@property
def mean_itl_ms(self) -> float:
if not self.inter_token_latencies:
return 0.0
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
def to_dict(self) -> dict[str, Any]:
d = asdict(self)
d["tpot_ms"] = self.tpot_ms
d["mean_itl_ms"] = self.mean_itl_ms
return d
# ---------------------------------------------------------------------------
# Health checker
# ---------------------------------------------------------------------------
class HealthChecker:
"""Periodically check service health and maintain a healthy-port list."""
def __init__(
self,
host: str,
ports: list[int],
check_interval: float = 5.0,
max_failures: int = 2,
recovery_interval: float = 10.0,
):
self.host = host
self.all_ports = ports
self.check_interval = check_interval
self.max_failures = max_failures
self.recovery_interval = recovery_interval
# port -> consecutive failure count
self._failures: dict[int, int] = {p: 0 for p in ports}
# port -> last healthy timestamp
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
self._lock = asyncio.Lock()
self._task: asyncio.Task | None = None
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
url = f"http://{self.host}:{port}/health"
try:
if HAS_AIOHTTP and session is not None:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
return resp.status == 200
else:
req = urllib.request.Request(url, method="GET")
with urllib.request.urlopen(req, timeout=5) as resp:
return resp.status == 200
except Exception:
return False
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
checks = [self._check_one(p, session) for p in self.all_ports]
results = await asyncio.gather(*checks, return_exceptions=True)
now = time.monotonic()
async with self._lock:
for port, ok in zip(self.all_ports, results):
if isinstance(ok, Exception):
ok = False
if ok:
self._failures[port] = 0
self._last_healthy[port] = now
else:
self._failures[port] += 1
async def start(self, session: aiohttp.ClientSession | None) -> None:
async def _loop() -> None:
while True:
await asyncio.sleep(self.check_interval)
await self._check_all(session)
self._task = asyncio.create_task(_loop())
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
@property
async def healthy_ports(self) -> list[int]:
now = time.monotonic()
async with self._lock:
healthy = []
for p in self.all_ports:
if self._failures[p] < self.max_failures:
healthy.append(p)
elif now - self._last_healthy[p] > self.recovery_interval:
# Give it another chance after recovery_interval.
self._failures[p] = max(0, self._failures[p] - 1)
healthy.append(p)
return healthy
async def is_healthy(self, port: int) -> bool:
ports = await self.healthy_ports
return port in ports
# ---------------------------------------------------------------------------
# Load balancer with failover
# ---------------------------------------------------------------------------
class LoadBalancer:
"""Distribute requests across healthy backend ports with auto-failover."""
def __init__(
self,
ports: list[int],
health_checker: HealthChecker,
strategy: str = "round_robin",
):
self.all_ports = ports
self.health_checker = health_checker
self.strategy = strategy
self._idx = 0
self._lock = asyncio.Lock()
def _next_round_robin(self, healthy: list[int]) -> int:
if not healthy:
return self.all_ports[self._idx % len(self.all_ports)]
for _ in range(len(self.all_ports)):
port = self.all_ports[self._idx % len(self.all_ports)]
self._idx += 1
if port in healthy:
return port
# Fallback: all unhealthy, pick first healthy anyway.
return healthy[0] if healthy else self.all_ports[0]
def _next_random(self, healthy: list[int]) -> int:
if healthy:
return random.choice(healthy)
return random.choice(self.all_ports)
async def next_port(self) -> int:
healthy = await self.health_checker.healthy_ports
async with self._lock:
if self.strategy == "round_robin":
return self._next_round_robin(healthy)
elif self.strategy in ("random", "least_conn"):
return self._next_random(healthy)
else:
return self._next_round_robin(healthy)
@property
def port_count(self) -> int:
return len(self.all_ports)
# ---------------------------------------------------------------------------
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
tokens.extend(words)
return " ".join(tokens[:needed])
# ---------------------------------------------------------------------------
# aiohttp-based async request (preferred)
# ---------------------------------------------------------------------------
async def send_request_aiohttp(
session: aiohttp.ClientSession,
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
headers = {"Content-Type": "application/json"}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
first_token_received = False
last_token_time = 0.0
try:
async with session.post(url, json=payload, headers=headers) as resp:
if resp.status != 200:
text = await resp.text()
result.error = f"HTTP {resp.status}: {text[:200]}"
return result
async for line in resp.content:
line = line.decode("utf-8").strip()
if not line or not line.startswith("data: "):
continue
data_str = line[len("data: "):]
if data_str == "[DONE]":
break
try:
chunk = json.loads(data_str)
except json.JSONDecodeError:
continue
choices = chunk.get("choices", [])
if not choices:
continue
delta = choices[0].get("delta", {})
content = delta.get("content", "")
if content:
now = time.perf_counter()
if not first_token_received:
result.first_token_time_ms = (now - start_time) * 1000
first_token_received = True
else:
result.inter_token_latencies.append((now - last_token_time) * 1000)
last_token_time = now
result.output_tokens += 1
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.input_tokens = len(prompt.split()) # approximate
result.success = True
except asyncio.TimeoutError:
result.error = "timeout"
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# urllib-based synchronous request (fallback, no aiohttp)
# ---------------------------------------------------------------------------
def send_request_urllib(
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": False,
}
data = json.dumps(payload).encode("utf-8")
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
try:
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=600) as resp:
body = json.loads(resp.read().decode("utf-8"))
choices = body.get("choices", [])
if not choices:
result.error = "no choices in response"
return result
content = choices[0].get("message", {}).get("content", "")
result.output_tokens = len(content.split()) if content else 0
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.first_token_time_ms = result.e2e_latency_ms
result.input_tokens = len(prompt.split())
result.success = True
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# Unified send_request wrapper with retry & failover
# ---------------------------------------------------------------------------
async def send_request_with_retry(
session: aiohttp.ClientSession | None,
host: str,
lb: LoadBalancer,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
max_retries: int = 2,
) -> RequestResult:
"""Send a request, retrying on failure with a different port each time."""
result = RequestResult(request_id=request_id)
for attempt in range(max_retries + 1):
port = await lb.next_port()
if HAS_AIOHTTP and session is not None:
result = await send_request_aiohttp(
session, host, port, model, request_id, prompt, max_tokens, temperature
)
else:
loop = asyncio.get_event_loop()
result = await loop.run_in_executor(
None, send_request_urllib,
host, port, model, request_id, prompt, max_tokens, temperature
)
result.retry_count = attempt
if result.success:
return result
# If failed, try another port on next iteration (unless last attempt).
if attempt < max_retries:
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
return result
# ---------------------------------------------------------------------------
# Benchmark orchestration
# ---------------------------------------------------------------------------
async def run_benchmark(
host: str,
ports: list[int],
lb: LoadBalancer,
health_checker: HealthChecker,
model: str,
concurrency: int,
input_len: int,
output_len: int,
num_prompts: int,
temperature: float,
output_file: str,
) -> dict[str, Any]:
"""Run the benchmark and write a JSONL file."""
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
results: list[RequestResult] = []
semaphore = asyncio.Semaphore(concurrency)
start = time.perf_counter()
if HAS_AIOHTTP:
connector = aiohttp.TCPConnector(limit=concurrency * 2)
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
await health_checker.start(session)
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
else:
await health_checker.start(None)
from concurrent.futures import ThreadPoolExecutor
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
None, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
duration = time.perf_counter() - start
# Write JSONL
with open(output_file, "w", encoding="utf-8") as f:
for r in results:
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
# Compute summary
successful = [r for r in results if r.success]
failed = len(results) - len(successful)
retried = [r for r in results if r.retry_count > 0]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
ttfts = [r.first_token_time_ms for r in successful]
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
e2es = [r.e2e_latency_ms for r in successful]
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
total_input_tokens = sum(r.input_tokens for r in successful)
total_output_tokens = sum(r.output_tokens for r in successful)
# Per-port breakdown
per_port_stats: dict[int, dict[str, Any]] = {}
for port in ports:
port_results = [r for r in successful if r.target_port == port]
port_failed = [r for r in results if r.target_port == port and not r.success]
if port_results or port_failed:
port_ttfts = [r.first_token_time_ms for r in port_results]
port_e2es = [r.e2e_latency_ms for r in port_results]
per_port_stats[port] = {
"count": len(port_results),
"failed": len(port_failed),
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
}
# Health-check stats
healthy_at_end = await health_checker.healthy_ports
summary = {
"completed": len(successful),
"failed": failed,
"retried": len(retried),
"duration": duration,
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
"total_input_tokens": total_input_tokens,
"total_output_tokens": total_output_tokens,
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
"median_ttft_ms": percentile(ttfts, 50),
"p90_ttft_ms": percentile(ttfts, 90),
"p95_ttft_ms": percentile(ttfts, 95),
"p99_ttft_ms": percentile(ttfts, 99),
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
"median_tpot_ms": percentile(tpots, 50),
"p90_tpot_ms": percentile(tpots, 90),
"p95_tpot_ms": percentile(tpots, 95),
"p99_tpot_ms": percentile(tpots, 99),
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
"median_e2e_latency_ms": percentile(e2es, 50),
"p90_e2e_latency_ms": percentile(e2es, 90),
"p95_e2e_latency_ms": percentile(e2es, 95),
"p99_e2e_latency_ms": percentile(e2es, 99),
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
"median_itl_ms": percentile(itls, 50),
"p90_itl_ms": percentile(itls, 90),
"p95_itl_ms": percentile(itls, 95),
"p99_itl_ms": percentile(itls, 99),
"per_port_stats": per_port_stats,
"healthy_ports_at_end": healthy_at_end,
"lb_strategy": lb.strategy,
"num_services": lb.port_count,
}
# Append summary as the last line
with open(output_file, "a", encoding="utf-8") as f:
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
return summary
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=30005,
help="Single service port (backward compatible)")
parser.add_argument("--ports", default="",
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
parser.add_argument("--model", default="deepseek-v4-flash")
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--input-len", type=int, default=512)
parser.add_argument("--output-len", type=int, default=256)
parser.add_argument("--num-prompts", type=int, default=10)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--lb-strategy", default="round_robin",
choices=["round_robin", "random", "least_conn"],
help="Load balancing strategy across services")
parser.add_argument("--health-check-interval", type=float, default=5.0,
help="Seconds between health checks (default: 5)")
parser.add_argument("--health-max-failures", type=int, default=2,
help="Consecutive failures before marking a port unhealthy (default: 2)")
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
help="Seconds before retrying an unhealthy port (default: 10)")
parser.add_argument("--request-max-retries", type=int, default=2,
help="Max retries per request on failure (default: 2)")
parser.add_argument("--output-file", required=True)
args = parser.parse_args()
# Determine ports: --ports takes precedence, else fall back to --port.
if args.ports:
ports = [int(p.strip()) for p in args.ports.split(",")]
else:
ports = [args.port]
health_checker = HealthChecker(
host=args.host,
ports=ports,
check_interval=args.health_check_interval,
max_failures=args.health_max_failures,
recovery_interval=args.health_recovery_interval,
)
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
summary = asyncio.run(run_benchmark(
host=args.host,
ports=ports,
lb=lb,
health_checker=health_checker,
model=args.model,
concurrency=args.concurrency,
input_len=args.input_len,
output_len=args.output_len,
num_prompts=args.num_prompts,
temperature=args.temperature,
output_file=args.output_file,
))
print(json.dumps(summary, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()

84
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env
View File

@ -1,84 +0,0 @@
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
#
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
# Service 1: port 30005, GPUs 0,1
# Service 2: port 30006, GPUs 2,3
# Service 3: port 30007, GPUs 4,5
# Service 4: port 30008, GPUs 6,7
#
# bench_client.py distributes requests across all 4 services via round-robin
# load balancing, giving accurate cluster-wide throughput even at low
# per-service concurrency.
EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench_no_fp8"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
# Primary port (backward compatible); multi-service ports are in PORTS.
PORT="30005"
PORTS="30005,30006,30007,30008"
BACKEND="vllm"
ENGINE="vllm"
# Native virtual environments on the host.
VENV_SERVER="/data/user1/yy/envs/vllm"
VENV_CLIENT="/data/user1/yy/envs/vllm"
# Load balancing strategy: round_robin | random | least_conn
LB_STRATEGY="round_robin"
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
# For multi-service we include low-concurrency scenarios to show how
# requests spread across 4 services.
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
)
# Server start script bundled with this experiment.
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

206
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py
View File

@ -1,206 +0,0 @@
#!/usr/bin/env python3
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
Reads JSONL files produced by bench_client.py (one request per line,
last line is the summary) and generates:
- results.json (appended scenarios, following the standard schema)
- report.md (human-readable summary)
Usage:
python3 parse_results.py <result_root>
"""
import json
import sys
from pathlib import Path
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
"""Read the summary JSON (last line) and all per-request lines."""
requests: list[dict] = []
summary: dict | None = None
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
# The summary line has "completed" and "duration" keys.
if "completed" in obj and "duration" in obj:
summary = obj
else:
requests.append(obj)
return summary, requests
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
completed = summary.get("completed", 0)
total = len(requests)
failed = total - completed if total > 0 else 0
duration_s = summary.get("duration", 0.0)
# Extract per-request metrics for percentiles.
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
return {
"success": completed,
"failed": failed,
"duration_s": duration_s,
"request_throughput": summary.get("request_throughput", 0.0),
"input_token_throughput": summary.get("input_throughput", 0.0),
"output_token_throughput": summary.get("output_throughput", 0.0),
"total_token_throughput": summary.get("total_throughput", 0.0),
"total_input_tokens": summary.get("total_input_tokens", 0),
"total_output_tokens": summary.get("total_output_tokens", 0),
"e2e_ms": {
"mean": summary.get("mean_e2e_latency_ms", 0.0),
"p50": percentile(e2es, 50),
"p90": percentile(e2es, 90),
"p95": percentile(e2es, 95),
"p99": percentile(e2es, 99),
},
"ttft_ms": {
"mean": summary.get("mean_ttft_ms", 0.0),
"p50": percentile(ttfts, 50),
"p90": percentile(ttfts, 90),
"p95": percentile(ttfts, 95),
"p99": percentile(ttfts, 99),
},
"tpot_ms": {
"mean": summary.get("mean_tpot_ms", 0.0),
"p50": percentile(tpots, 50),
"p90": percentile(tpots, 90),
"p95": percentile(tpots, 95),
"p99": percentile(tpots, 99),
},
"itl_ms": {
"mean": summary.get("mean_itl_ms", 0.0),
"p50": percentile(itls, 50),
"p90": percentile(itls, 90),
"p95": percentile(itls, 95),
"p99": percentile(itls, 99),
},
}
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
return f"c{concurrency}_i{input_len}_o{output_len}"
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
if ttft_ok and tpot_ok:
mark = "✅"
elif ttft_ok or tpot_ok:
mark = "⚠️"
else:
mark = "❌"
return {
"ttft_p95_ok": ttft_ok,
"tpot_mean_ok": tpot_ok,
"overall": mark,
}
def append_scenario(results_json: Path, scenario: dict) -> None:
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"].append(scenario)
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
report_path = result_root / "report.md"
with open(report_path, "w", encoding="utf-8") as f:
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
f.write(f"- **Result root**: `{result_root}`\n\n")
f.write("## Results\n\n")
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
for s in scenarios:
cfg = s["config"]
m = s["metrics"]
slo = s.get("slo_status", {}).get("overall", "")
f.write(
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['total_token_throughput']:.2f} | "
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
)
f.write("\n")
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
def main() -> None:
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
raw_dir = result_root / "raw_outputs"
results_json = result_root / "results.json"
if not raw_dir.exists():
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
scenarios = []
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
parts = jsonl_path.stem.split("_")
if len(parts) < 5:
continue
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
summary, requests = parse_jsonl(jsonl_path)
if summary is None:
continue
metrics = compute_metrics(summary, requests)
scenario = {
"name": scenario_name(concurrency, input_len, output_len),
"config": {
"concurrency": concurrency,
"input_len": input_len,
"output_len": output_len,
"dataset": "random",
"num_prompts": metrics["success"] + metrics["failed"],
},
"metrics": metrics,
"slo_status": slo_status(metrics),
"raw_file": str(jsonl_path),
}
scenarios.append(scenario)
if not scenarios:
print("No benchmark outputs found to parse")
return
if results_json.exists():
for s in scenarios:
append_scenario(results_json, s)
generate_report(result_root, scenarios)
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
if __name__ == "__main__":
main()

47
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json
View File

@ -1,47 +0,0 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091141",
"timestamp": "2026-07-10T09:11:41+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},
"config": {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20",
"4 512 256 80",
"8 512 256 160",
"16 512 256 320",
"32 512 256 640",
"64 512 256 1280",
"128 512 256 2560",
"1 2048 512 20",
"8 2048 512 160",
"32 2048 512 640",
"128 2048 512 2560",
"1 4096 1024 20",
"8 4096 1024 160",
"32 4096 1024 640",
"128 4096 1024 2560"
]
},
"scenarios": []
}

47
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json
View File

@ -1,47 +0,0 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091522",
"timestamp": "2026-07-10T09:15:22+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},
"config": {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20",
"4 512 256 80",
"8 512 256 160",
"16 512 256 320",
"32 512 256 640",
"64 512 256 1280",
"128 512 256 2560",
"1 2048 512 20",
"8 2048 512 160",
"32 2048 512 640",
"128 2048 512 2560",
"1 4096 1024 20",
"8 4096 1024 160",
"32 4096 1024 640",
"128 4096 1024 2560"
]
},
"scenarios": []
}

47
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json
View File

@ -1,47 +0,0 @@
{
"metadata": {
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
"run_id": "20260710-091700",
"timestamp": "2026-07-10T09:17:00+00:00",
"model": "/data/models/DeepSeek-V4-Flash",
"backend": "vllm",
"engine": "vllm",
"hardware": "8x NVIDIA H200 143GB",
"accelerator": "NVIDIA H200",
"chip": "nvidia_h200",
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
"env": "/data/user1/yy/envs/vllm",
"git_commit": "39fb076",
"git_dirty": "dirty",
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
},
"config": {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20",
"4 512 256 80",
"8 512 256 160",
"16 512 256 320",
"32 512 256 640",
"64 512 256 1280",
"128 512 256 2560",
"1 2048 512 20",
"8 2048 512 160",
"32 2048 512 640",
"128 2048 512 2560",
"1 4096 1024 20",
"8 4096 1024 160",
"32 4096 1024 640",
"128 4096 1024 2560"
]
},
"scenarios": []
}

277
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh
View File

@ -1,277 +0,0 @@
#!/usr/bin/env bash
# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200.
#
# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7).
# bench_client.py distributes requests across all 4 services via round-robin LB,
# giving accurate cluster-wide throughput even at low per-service concurrency.
#
# Usage:
# bash run_bench.sh
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
RAW_DIR="${RESULT_ROOT}/raw_outputs"
LOG_DIR="${RESULT_ROOT}/logs"
ensure_result_root "$RESULT_ROOT"
log_init "${LOG_DIR}/orchestrator.log"
log "experiment=${EXPERIMENT_NAME}"
log "run_id=${RUN_ID}"
log "result_root=${RESULT_ROOT}"
log "platform=${PLATFORM}"
log "chip=${CHIP}"
log "accelerator=${ACCELERATOR}"
log "engine=${ENGINE}"
log "hardware=${HARDWARE}"
log "model=${MODEL_PATH}"
log "services=${PORTS}"
log "lb_strategy=${LB_STRATEGY}"
log "server_start_script=${SERVER_START_SCRIPT}"
# Write initial metadata for this run.
METADATA_JSON="${RESULT_ROOT}/results.json"
write_metadata_json \
"$METADATA_JSON" \
"$EXPERIMENT_NAME" \
"$RUN_ID" \
"$MODEL_PATH" \
"$BACKEND" \
"$ENGINE" \
"$HARDWARE" \
"$ACCELERATOR" \
"$CHIP" \
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
"$VENV_SERVER" \
"H200 4x vLLM TP=2 multi-service (no FP8 KV cache) benchmark using bench_client.py"
# Update metadata with config.
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
import json, sys
path = sys.argv[1]
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
data["config"] = {
"tp": 2,
"num_services": 4,
"ports": "30005,30006,30007,30008",
"gpu_pairs": "0,1|2,3|4,5|6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20", "4 512 256 80", "8 512 256 160",
"16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560",
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
PY
# ---------------------------------------------------------------------------
# Server helpers (multi-service)
# ---------------------------------------------------------------------------
is_server_healthy() {
local port="$1"
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
}
are_all_services_healthy() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
for p in "${PORT_LIST[@]}"; do
if ! is_server_healthy "$p"; then
return 1
fi
done
return 0
}
stop_all_servers() {
log "stopping all vllm services"
# Kill by PID files first.
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
[[ -f "$pid_file" ]] || continue
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "killing pid=${pid}"
kill "$pid" 2>/dev/null || true
sleep 2
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
done
# Fallback: kill any remaining vllm processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
}
start_all_servers() {
log "starting all vllm services with ${SERVER_START_SCRIPT}"
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
exit 1
fi
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
if ! are_all_services_healthy "$PORTS"; then
log "error: not all vllm services became healthy"
exit 1
fi
log "all vllm services are healthy"
}
# ---------------------------------------------------------------------------
# Pre-benchmark health check with detailed reporting
# ---------------------------------------------------------------------------
check_services_detailed() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy_count=0
local unhealthy_ports=""
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
((healthy_count++))
else
unhealthy_ports="${unhealthy_ports}${p} "
fi
done
echo "$healthy_count"
if [[ -n "$unhealthy_ports" ]]; then
echo "unhealthy: ${unhealthy_ports}" >&2
fi
}
wait_for_all_services() {
local ports="$1"
local max_wait="${2:-240}"
local interval="${3:-5}"
local elapsed=0
while (( elapsed < max_wait )); do
local healthy_count
healthy_count="$(check_services_detailed "$ports" | head -1)"
IFS=',' read -ra PORT_LIST <<< "$ports"
if (( healthy_count == ${#PORT_LIST[@]} )); then
return 0
fi
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
sleep "$interval"
((elapsed += interval))
done
return 1
}
# ---------------------------------------------------------------------------
# Mid-benchmark health check (called between scenarios)
# ---------------------------------------------------------------------------
check_and_report_services() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy=()
local unhealthy=()
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
healthy+=("$p")
else
unhealthy+=("$p")
fi
done
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
if [[ ${#unhealthy[@]} -gt 0 ]]; then
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
fi
return ${#unhealthy[@]}
}
on_exit() {
local code=$?
log "orchestrator exiting with code=${code}"
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
stop_all_servers
fi
exit "$code"
}
trap on_exit EXIT
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
if ! wait_for_all_services "$PORTS"; then
log "error: not all healthy services found on ports ${PORTS}"
exit 1
fi
else
stop_all_servers
start_all_servers
fi
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
--host 127.0.0.1 \
--ports "$PORTS" \
--model "$SERVED_MODEL_NAME" \
--concurrency "$concurrency" \
--input-len "$input_len" \
--output-len "$output_len" \
--num-prompts "$num_prompts" \
--lb-strategy "$LB_STRATEGY" \
--health-check-interval 5 \
--health-max-failures 2 \
--health-recovery-interval 10 \
--request-max-retries 2 \
--output-file "$output_file" \
> "$detail_log" 2>&1 || {
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
# After failure, check service health before continuing.
check_and_report_services "$PORTS"
continue
}
log "finished scenario: output=${output_file}"
# Between scenarios, report service health so we can spot degradation early.
check_and_report_services "$PORTS" || true
done
log "===== BENCHMARK DONE ====="
log "parsing results"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
}
log "all results saved to ${RESULT_ROOT}"

109
experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh
View File

@ -1,109 +0,0 @@
#!/bin/bash
# Start 4 independent vLLM TP=2 services on 8x H200.
# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7).
# This maximizes GPU utilization for low-concurrency workloads.
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
VENV="${VENV_SERVER}"
export PATH="$VENV/bin:$PATH"
VLLM="$VENV/bin/vllm"
TP=2
LOG_DIR="${SCRIPT_DIR}/logs"
mkdir -p "$LOG_DIR"
# 4 services, each on 2 GPUs.
# Port and GPU pair mapping.
SERVICES=(
"30005:0,1"
"30006:2,3"
"30007:4,5"
"30008:6,7"
)
# Kill any existing vLLM processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
# Remove old PID files.
rm -f "${SCRIPT_DIR}"/*.pid
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
GPUS="${svc##*:}"
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
echo "Log: ${LOG}"
CUDA_VISIBLE_DEVICES="${GPUS}" \
nohup "$VLLM" serve "$MODEL_PATH" \
--trust-remote-code \
--tensor-parallel-size "$TP" \
--block-size 256 \
--served-model-name "$SERVED_MODEL_NAME" \
--port "$PORT" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
done
echo ""
echo "Waiting for all 4 services to become healthy..."
MAX_WAIT=240
all_healthy=0
for i in $(seq 1 $MAX_WAIT); do
all_healthy=1
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
all_healthy=0
break
fi
done
if [[ "$all_healthy" -eq 1 ]]; then
echo "All 4 services are healthy!"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
echo " - http://127.0.0.1:${PORT}"
done
exit 0
fi
# Check if any process died early.
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
if [[ -f "$PID_FILE" ]]; then
PID="$(cat "$PID_FILE")"
if ! kill -0 "$PID" 2>/dev/null; then
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
tail -200 $LOG 2>/dev/null || true
exit 1
fi
fi
done
if (( i % 10 == 0 )); then
echo "Waiting... (${i}/${MAX_WAIT})"
fi
sleep 5
done
echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
echo "--- Last 50 lines of port ${PORT} ---"
tail -50 $LOG 2>/dev/null || true
done
exit 1

53
experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py
View File

@ -228,16 +228,22 @@ class LoadBalancer:
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
import uuid
def make_prompt(token_len: int) -> str:
prefix = str(uuid.uuid4())
words = [
"the", "quick", "brown", "fox",
"jumps", "over", "lazy", "dog",
"benchmark", "performance", "latency"
]
tokens = [prefix]
while len(tokens) < token_len:
tokens.extend(words)
return " ".join(tokens[:needed])
return " ".join(tokens[:token_len])
# ---------------------------------------------------------------------------
@ -440,6 +446,35 @@ async def run_benchmark(
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
print("Starting warmup...")
warmup_per_service = 10
warmup_requests = warmup_per_service * len(ports)
warmup_prompts = [
make_prompt(input_len)
for _ in range(warmup_requests)
]
warmup_tasks = [
send_request_with_retry(
session=session,
host=host,
lb=lb,
model=model,
request_id=-1,
prompt=p,
max_tokens=output_len,
temperature=temperature,
)
for p in warmup_prompts
]
await asyncio.gather(*warmup_tasks)
print("Warmup finished.")
# 给 CUDA / Scheduler 一点时间稳定
await asyncio.sleep(2)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)

184
experiments/dsv4_h200_vllm_tp4_custom_bench/config.env
View File

@ -34,50 +34,146 @@ LB_STRATEGY="round_robin"
# For multi-service we include low-concurrency scenarios to show how
# requests spread across 2 services.
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 1024 128 20"
"1 1024 256 20"
"1 1024 512 20"
"1 1024 1024 20"
"1 1024 2048 20"
"1 1024 4096 20"
"8 1024 128 160"
"8 1024 256 160"
"8 1024 512 160"
"8 1024 1024 160"
"8 1024 2048 160"
"8 1024 4096 160"
"32 1024 128 640"
"32 1024 256 640"
"32 1024 512 640"
"32 1024 1024 640"
"32 1024 2048 640"
"32 1024 4096 640"
"128 1024 128 2560"
"128 1024 256 2560"
"128 1024 512 2560"
"128 1024 1024 2560"
"128 1024 2048 2560"
"128 1024 4096 2560"
"1 2048 128 20"
"1 2048 256 20"
"1 2048 512 20"
"1 2048 1024 20"
"1 2048 2048 20"
"1 2048 4096 20"
"8 2048 128 160"
"8 2048 256 160"
"8 2048 512 160"
"8 2048 1024 160"
"8 2048 2048 160"
"8 2048 4096 160"
"32 2048 128 640"
"32 2048 256 640"
"32 2048 512 640"
"32 2048 1024 640"
"32 2048 2048 640"
"32 2048 4096 640"
"128 2048 128 2560"
"128 2048 256 2560"
"128 2048 512 2560"
"128 2048 1024 2560"
"128 2048 2048 2560"
"128 2048 4096 2560"
"1 4096 128 20"
"1 4096 256 20"
"1 4096 512 20"
"1 4096 1024 20"
"1 4096 2048 20"
"1 4096 4096 20"
"8 4096 128 160"
"8 4096 256 160"
"8 4096 512 160"
"8 4096 1024 160"
"8 4096 2048 160"
"8 4096 4096 160"
"32 4096 128 640"
"32 4096 256 640"
"32 4096 512 640"
"32 4096 1024 640"
"32 4096 2048 640"
"32 4096 4096 640"
"128 4096 128 2560"
"128 4096 256 2560"
"128 4096 512 2560"
"128 4096 1024 2560"
"128 4096 2048 2560"
"128 4096 4096 2560"
"1 16384 128 20"
"1 16384 256 20"
"1 16384 512 20"
"1 16384 1024 20"
"1 16384 2048 20"
"8 16384 128 160"
"8 16384 256 160"
"8 16384 512 160"
"8 16384 1024 160"
"8 16384 2048 160"
"32 16384 128 640"
"32 16384 256 640"
"32 16384 512 640"
"32 16384 1024 640"
"32 16384 2048 640"
"128 16384 128 2560"
"128 16384 256 2560"
"128 16384 512 2560"
"128 16384 1024 2560"
"128 16384 2048 2560"
"1 65536 128 20"
"1 65536 256 20"
"1 65536 512 20"
"1 65536 1024 20"
"8 65536 128 160"
"8 65536 256 160"
"8 65536 512 160"
"8 65536 1024 160"
"32 65536 1024 160"
"1 131072 128 20"
"1 131072 256 20"
"1 131072 512 20"
"8 131072 512 160"
"1 262144 256 20"
"8 262144 128 160"
"1 524288 128 20"
)
# Server start script bundled with this experiment.

129
experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md Normal file
View File

@ -0,0 +1,129 @@
# H200 vLLM TP=2 Custom Benchmark Report
- **Client**: `bench_client.py` (async OpenAI API, per-request timing)
- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)
- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908`
## Results
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
| c128_i1024_o1024 | 128 | 1024 | 1024 | 172.39 | 2560 | 0 | 14.85 | 15206.17 | 3100.60 | 18306.76 | 267.16 | 478.80 | 2297.70 | 38.53 | 66.16 | 70.07 | 8265.00 | 13967.23 | 17536.46 | ✅ |
| c128_i1024_o128 | 128 | 1024 | 128 | 98.84 | 2560 | 0 | 25.90 | 26522.07 | 3314.91 | 29836.98 | 900.39 | 1590.87 | 2307.93 | 30.16 | 35.98 | 37.09 | 4730.89 | 5530.64 | 6122.15 | ✅ |
| c128_i1024_o2048 | 128 | 1024 | 2048 | 161.76 | 2560 | 0 | 15.83 | 16205.28 | 3297.74 | 19503.02 | 256.61 | 340.87 | 2315.99 | 36.07 | 39.27 | 40.30 | 7732.01 | 10616.59 | 11870.42 | ✅ |
| c128_i1024_o256 | 128 | 1024 | 256 | 161.16 | 2560 | 0 | 15.89 | 16266.52 | 3250.72 | 19517.24 | 259.69 | 356.25 | 2333.30 | 36.68 | 40.81 | 51.99 | 7727.17 | 10133.95 | 11225.21 | ✅ |
| c128_i1024_o4096 | 128 | 1024 | 4096 | 163.16 | 2560 | 0 | 15.69 | 16066.27 | 3281.03 | 19347.31 | 257.28 | 422.58 | 2303.92 | 36.09 | 39.21 | 40.35 | 7768.07 | 10730.88 | 12234.15 | ✅ |
| c128_i1024_o512 | 128 | 1024 | 512 | 161.49 | 2560 | 0 | 15.85 | 16232.48 | 3275.92 | 19508.40 | 255.16 | 334.67 | 2297.21 | 36.27 | 39.39 | 40.57 | 7716.22 | 10581.99 | 11832.60 | ✅ |
| c128_i16384_o1024 | 128 | 16384 | 1024 | 919.25 | 2560 | 0 | 2.78 | 45627.36 | 697.33 | 46324.69 | 2625.93 | 3630.11 | 33634.45 | 171.30 | 196.69 | 207.71 | 45001.23 | 90438.26 | 123707.69 | ❌ |
| c128_i16384_o128 | 128 | 16384 | 128 | 859.37 | 2560 | 0 | 2.98 | 48806.72 | 380.95 | 49187.68 | 2833.19 | 3131.25 | 33643.84 | 311.88 | 328.04 | 328.10 | 42403.99 | 44239.06 | 73426.74 | ❌ |
| c128_i16384_o2048 | 128 | 16384 | 2048 | 918.61 | 2560 | 0 | 2.79 | 45659.28 | 701.99 | 46361.27 | 2601.36 | 3607.43 | 33624.66 | 170.03 | 195.00 | 208.79 | 45030.71 | 92401.43 | 124284.89 | ❌ |
| c128_i16384_o256 | 128 | 16384 | 256 | 897.62 | 2560 | 0 | 2.85 | 46726.87 | 602.42 | 47329.29 | 2804.71 | 4596.59 | 33617.37 | 197.06 | 224.66 | 236.73 | 44219.84 | 57755.16 | 76587.71 | ❌ |
| c128_i16384_o512 | 128 | 16384 | 512 | 915.13 | 2560 | 0 | 2.80 | 45833.00 | 684.56 | 46517.56 | 2646.03 | 3737.57 | 33718.32 | 174.28 | 203.88 | 224.11 | 44914.54 | 88267.88 | 100313.27 | ❌ |
| c128_i2048_o1024 | 128 | 2048 | 1024 | 204.30 | 2560 | 0 | 12.53 | 25663.20 | 2671.39 | 28334.59 | 372.84 | 493.00 | 4208.03 | 44.24 | 48.11 | 50.53 | 9762.62 | 15935.84 | 21541.33 | ✅ |
| c128_i2048_o128 | 128 | 2048 | 128 | 153.29 | 2560 | 0 | 16.70 | 34202.45 | 2125.54 | 36327.99 | 791.19 | 1283.06 | 4199.42 | 52.51 | 58.15 | 59.97 | 7426.54 | 8226.85 | 10576.91 | ⚠️ |
| c128_i2048_o2048 | 128 | 2048 | 2048 | 208.80 | 2560 | 0 | 12.26 | 25109.31 | 2631.24 | 27740.55 | 365.94 | 530.02 | 4211.56 | 44.40 | 48.85 | 54.00 | 9842.38 | 16285.06 | 20408.42 | ✅ |
| c128_i2048_o256 | 128 | 2048 | 256 | 196.63 | 2560 | 0 | 13.02 | 26663.63 | 2605.54 | 29269.17 | 379.32 | 522.39 | 4205.72 | 45.73 | 50.01 | 51.66 | 9476.10 | 12538.01 | 13720.71 | ✅ |
| c128_i2048_o4096 | 128 | 2048 | 4096 | 204.98 | 2560 | 0 | 12.49 | 25578.03 | 2696.32 | 28274.35 | 369.65 | 497.47 | 4203.66 | 44.06 | 47.90 | 49.49 | 9834.14 | 15705.61 | 22444.28 | ✅ |
| c128_i2048_o512 | 128 | 2048 | 512 | 205.58 | 2560 | 0 | 12.45 | 25502.28 | 2682.24 | 28184.52 | 364.96 | 479.81 | 4198.31 | 44.32 | 48.38 | 49.46 | 9849.41 | 16273.27 | 21537.21 | ✅ |
| c128_i4096_o1024 | 128 | 4096 | 1024 | 299.56 | 2560 | 0 | 8.55 | 35004.04 | 1823.49 | 36827.53 | 670.94 | 898.57 | 8472.05 | 64.95 | 71.72 | 78.31 | 14410.80 | 23823.73 | 34763.30 | ⚠️ |
| c128_i4096_o128 | 128 | 4096 | 128 | 252.68 | 2560 | 0 | 10.13 | 41497.56 | 1283.45 | 42781.02 | 1110.40 | 1837.39 | 8464.33 | 89.25 | 99.28 | 101.99 | 12341.74 | 13909.51 | 19709.73 | ⚠️ |
| c128_i4096_o2048 | 128 | 4096 | 2048 | 299.51 | 2560 | 0 | 8.55 | 35009.54 | 1841.89 | 36851.42 | 673.64 | 903.51 | 8456.42 | 64.31 | 71.20 | 77.98 | 14467.32 | 24917.88 | 37076.16 | ⚠️ |
| c128_i4096_o256 | 128 | 4096 | 256 | 288.98 | 2560 | 0 | 8.86 | 36284.85 | 1743.96 | 38028.81 | 688.49 | 1029.32 | 8474.33 | 68.27 | 74.67 | 80.07 | 14052.64 | 18727.14 | 21844.01 | ⚠️ |
| c128_i4096_o4096 | 128 | 4096 | 4096 | 297.69 | 2560 | 0 | 8.60 | 35223.60 | 1820.84 | 37044.43 | 679.88 | 979.68 | 8471.74 | 65.18 | 72.12 | 79.04 | 14368.53 | 24055.68 | 33864.98 | ⚠️ |
| c128_i4096_o512 | 128 | 4096 | 512 | 296.90 | 2560 | 0 | 8.62 | 35317.89 | 1815.79 | 37133.68 | 666.19 | 888.77 | 8469.32 | 65.45 | 72.23 | 78.81 | 14377.25 | 24981.06 | 33674.28 | ⚠️ |
| c128_i512_o256 | 128 | 512 | 256 | 155.29 | 2560 | 0 | 16.49 | 8440.72 | 3401.04 | 11841.77 | 216.12 | 439.65 | 1351.98 | 35.17 | 38.71 | 39.83 | 7433.74 | 9800.36 | 10039.98 | ✅ |
| c16_i512_o256 | 16 | 512 | 256 | 60.82 | 320 | 0 | 5.26 | 2693.64 | 1089.67 | 3783.32 | 145.56 | 311.20 | 388.94 | 12.62 | 17.00 | 17.83 | 2748.60 | 4105.76 | 4640.23 | ✅ |
| c1_i1024_o1024 | 1 | 1024 | 1024 | 52.34 | 20 | 0 | 0.38 | 391.29 | 85.14 | 476.42 | 115.81 | 118.26 | 140.50 | 6.99 | 7.00 | 7.00 | 1666.63 | 2301.13 | 2358.69 | ✅ |
| c1_i1024_o128 | 1 | 1024 | 128 | 32.71 | 20 | 0 | 0.61 | 626.06 | 78.07 | 704.13 | 115.30 | 122.94 | 128.47 | 6.96 | 6.97 | 7.01 | 997.08 | 1004.84 | 1011.97 | ✅ |
| c1_i1024_o2048 | 1 | 1024 | 2048 | 48.50 | 20 | 0 | 0.41 | 422.30 | 83.02 | 505.31 | 115.62 | 119.50 | 126.29 | 6.99 | 7.00 | 7.01 | 1516.37 | 1700.77 | 1778.35 | ✅ |
| c1_i1024_o256 | 1 | 1024 | 256 | 48.48 | 20 | 0 | 0.41 | 422.40 | 81.26 | 503.66 | 114.81 | 120.58 | 125.23 | 6.99 | 7.01 | 7.01 | 1485.39 | 1837.55 | 1880.36 | ✅ |
| c1_i1024_o4096 | 1 | 1024 | 4096 | 51.98 | 20 | 0 | 0.38 | 393.97 | 82.72 | 476.69 | 116.51 | 126.81 | 137.83 | 6.99 | 7.00 | 7.01 | 1612.52 | 2093.77 | 2163.97 | ✅ |
| c1_i1024_o512 | 1 | 1024 | 512 | 49.68 | 20 | 0 | 0.40 | 412.23 | 78.90 | 491.13 | 132.68 | 174.48 | 175.39 | 7.00 | 7.02 | 7.02 | 1497.09 | 2052.55 | 2150.57 | ✅ |
| c1_i131072_o128 | 1 | 131072 | 128 | 257.45 | 20 | 0 | 0.08 | 10182.44 | 8.90 | 10191.34 | 6471.48 | 6574.88 | 6590.68 | 7.21 | 7.25 | 7.32 | 7290.28 | 7408.33 | 7425.83 | ⚠️ |
| c1_i131072_o256 | 1 | 131072 | 256 | 259.43 | 20 | 0 | 0.08 | 10104.51 | 13.11 | 10117.61 | 6523.80 | 6605.21 | 6683.36 | 7.25 | 7.28 | 7.30 | 7749.51 | 8208.24 | 8219.55 | ⚠️ |
| c1_i131072_o512 | 1 | 131072 | 512 | 281.03 | 20 | 0 | 0.07 | 9328.01 | 15.40 | 9343.41 | 6520.17 | 6595.29 | 6639.39 | 7.26 | 7.27 | 7.29 | 8083.13 | 9930.97 | 10147.71 | ⚠️ |
| c1_i16384_o1024 | 1 | 16384 | 1024 | 85.55 | 20 | 0 | 0.23 | 3830.19 | 66.12 | 3896.31 | 761.59 | 767.17 | 767.57 | 7.09 | 7.11 | 7.11 | 2761.83 | 4275.72 | 5529.53 | ✅ |
| c1_i16384_o128 | 1 | 16384 | 128 | 54.06 | 20 | 0 | 0.37 | 6061.09 | 46.98 | 6108.07 | 758.98 | 762.04 | 764.60 | 7.03 | 7.04 | 7.04 | 1645.31 | 1649.67 | 1651.54 | ✅ |
| c1_i16384_o2048 | 1 | 16384 | 2048 | 90.20 | 20 | 0 | 0.22 | 3632.96 | 62.71 | 3695.67 | 762.25 | 769.13 | 769.49 | 7.10 | 7.11 | 7.11 | 2762.54 | 3858.26 | 4063.08 | ✅ |
| c1_i16384_o256 | 1 | 16384 | 256 | 73.13 | 20 | 0 | 0.27 | 4480.86 | 56.19 | 4537.04 | 757.24 | 769.18 | 772.94 | 7.09 | 7.11 | 7.11 | 2205.64 | 2558.68 | 2568.89 | ✅ |
| c1_i16384_o512 | 1 | 16384 | 512 | 75.43 | 20 | 0 | 0.27 | 4344.23 | 53.96 | 4398.19 | 761.98 | 768.30 | 778.23 | 7.10 | 7.11 | 7.11 | 2198.96 | 2832.27 | 2889.46 | ✅ |
| c1_i2048_o1024 | 1 | 2048 | 1024 | 52.15 | 20 | 0 | 0.38 | 785.38 | 81.93 | 867.31 | 135.15 | 140.23 | 150.82 | 7.07 | 7.09 | 7.10 | 1639.00 | 2263.67 | 3129.84 | ✅ |
| c1_i2048_o128 | 1 | 2048 | 128 | 33.24 | 20 | 0 | 0.60 | 1232.22 | 74.97 | 1307.19 | 136.94 | 159.17 | 181.96 | 7.04 | 7.10 | 7.10 | 1006.62 | 1053.40 | 1075.84 | ✅ |
| c1_i2048_o2048 | 1 | 2048 | 2048 | 51.48 | 20 | 0 | 0.39 | 795.71 | 84.60 | 880.31 | 134.55 | 137.97 | 148.69 | 7.07 | 7.09 | 7.10 | 1667.40 | 3179.96 | 3366.92 | ✅ |
| c1_i2048_o256 | 1 | 2048 | 256 | 48.45 | 20 | 0 | 0.41 | 845.46 | 79.01 | 924.47 | 133.36 | 134.84 | 138.71 | 7.07 | 7.08 | 7.09 | 1479.65 | 1930.94 | 1931.75 | ✅ |
| c1_i2048_o4096 | 1 | 2048 | 4096 | 52.65 | 20 | 0 | 0.38 | 777.92 | 77.54 | 855.46 | 134.26 | 137.78 | 149.15 | 7.08 | 7.09 | 7.10 | 1571.48 | 2033.68 | 2095.42 | ✅ |
| c1_i2048_o512 | 1 | 2048 | 512 | 50.39 | 20 | 0 | 0.40 | 812.88 | 79.10 | 891.98 | 133.86 | 135.40 | 147.37 | 7.08 | 7.10 | 7.10 | 1537.04 | 2769.40 | 2894.64 | ✅ |
| c1_i262144_o256 | 1 | 262144 | 256 | 556.90 | 20 | 0 | 0.04 | 9414.44 | 6.27 | 9420.71 | 15234.61 | 15316.53 | 15320.27 | 7.47 | 7.55 | 7.57 | 16530.72 | 17025.05 | 17032.91 | ⚠️ |
| c1_i4096_o1024 | 1 | 4096 | 1024 | 50.77 | 20 | 0 | 0.39 | 1613.53 | 78.27 | 1691.81 | 223.98 | 228.84 | 229.98 | 7.08 | 7.09 | 7.09 | 1622.74 | 2090.63 | 2641.63 | ✅ |
| c1_i4096_o128 | 1 | 4096 | 128 | 36.18 | 20 | 0 | 0.55 | 2264.03 | 70.17 | 2334.20 | 223.37 | 225.71 | 227.21 | 7.03 | 7.04 | 7.07 | 1108.62 | 1117.89 | 1119.93 | ✅ |
| c1_i4096_o2048 | 1 | 4096 | 2048 | 54.09 | 20 | 0 | 0.37 | 1514.64 | 78.34 | 1592.98 | 223.30 | 226.85 | 227.97 | 7.08 | 7.09 | 7.09 | 1715.15 | 2736.41 | 2965.24 | ✅ |
| c1_i4096_o256 | 1 | 4096 | 256 | 53.93 | 20 | 0 | 0.37 | 1519.00 | 77.17 | 1596.18 | 224.11 | 227.58 | 229.35 | 7.07 | 7.09 | 7.09 | 1687.85 | 2022.93 | 2023.17 | ✅ |
| c1_i4096_o4096 | 1 | 4096 | 4096 | 50.60 | 20 | 0 | 0.40 | 1618.96 | 72.98 | 1691.94 | 223.43 | 228.43 | 229.73 | 7.08 | 7.09 | 7.09 | 1522.86 | 2146.74 | 2282.49 | ✅ |
| c1_i4096_o512 | 1 | 4096 | 512 | 59.75 | 20 | 0 | 0.33 | 1371.10 | 81.14 | 1452.25 | 224.35 | 229.07 | 230.84 | 7.07 | 7.08 | 7.08 | 1931.53 | 2977.28 | 3648.36 | ✅ |
| c1_i512_o256 | 1 | 512 | 256 | 53.26 | 20 | 0 | 0.38 | 192.26 | 80.81 | 273.07 | 167.28 | 203.13 | 205.33 | 6.99 | 7.02 | 7.03 | 1663.97 | 1951.39 | 1953.26 | ✅ |
| c1_i524288_o128 | 1 | 524288 | 128 | 1513.75 | 20 | 0 | 0.01 | 6927.01 | 1.20 | 6928.22 | 40602.84 | 41121.70 | 41219.49 | 7.78 | 7.92 | 7.93 | 41303.57 | 41857.14 | 41958.45 | ⚠️ |
| c1_i65536_o1024 | 1 | 65536 | 1024 | 152.81 | 20 | 0 | 0.13 | 8577.64 | 26.28 | 8603.92 | 2984.49 | 2994.56 | 3006.30 | 7.18 | 7.18 | 7.19 | 4418.27 | 5282.47 | 5593.15 | ✅ |
| c1_i65536_o128 | 1 | 65536 | 128 | 134.86 | 20 | 0 | 0.15 | 9718.82 | 17.89 | 9736.71 | 2996.65 | 3058.30 | 3078.26 | 7.11 | 7.16 | 7.16 | 3847.83 | 3911.31 | 3932.26 | ⚠️ |
| c1_i65536_o256 | 1 | 65536 | 256 | 153.56 | 20 | 0 | 0.13 | 8535.68 | 25.96 | 8561.64 | 2986.14 | 2994.62 | 3008.90 | 7.16 | 7.18 | 7.21 | 4407.24 | 4762.51 | 4778.90 | ✅ |
| c1_i65536_o512 | 1 | 65536 | 512 | 157.16 | 20 | 0 | 0.13 | 8340.12 | 27.70 | 8367.82 | 2986.06 | 2996.34 | 3023.15 | 7.18 | 7.19 | 7.20 | 4541.31 | 6107.66 | 6492.71 | ✅ |
| c2_i512_o256 | 2 | 512 | 256 | 44.94 | 40 | 0 | 0.89 | 455.73 | 188.32 | 644.06 | 123.41 | 177.91 | 187.26 | 7.06 | 7.48 | 7.68 | 1611.58 | 2020.21 | 2050.24 | ✅ |
| c32_i1024_o1024 | 32 | 1024 | 1024 | 78.93 | 640 | 0 | 8.11 | 8303.55 | 1684.20 | 9987.75 | 154.93 | 217.27 | 815.30 | 16.71 | 18.61 | 18.94 | 3611.27 | 4919.69 | 5510.54 | ✅ |
| c32_i1024_o128 | 32 | 1024 | 128 | 46.47 | 640 | 0 | 13.77 | 14103.72 | 1762.90 | 15866.62 | 408.25 | 714.24 | 757.98 | 13.41 | 16.99 | 17.83 | 2110.85 | 2431.17 | 2650.42 | ✅ |
| c32_i1024_o2048 | 32 | 1024 | 2048 | 81.00 | 640 | 0 | 7.90 | 8090.92 | 1622.21 | 9713.13 | 162.95 | 334.37 | 753.35 | 17.45 | 23.66 | 25.29 | 3730.41 | 5542.76 | 6244.95 | ✅ |
| c32_i1024_o256 | 32 | 1024 | 256 | 76.79 | 640 | 0 | 8.33 | 8534.95 | 1672.11 | 10207.06 | 154.72 | 212.02 | 753.62 | 16.97 | 19.11 | 20.00 | 3544.98 | 4746.50 | 4887.24 | ✅ |
| c32_i1024_o4096 | 32 | 1024 | 4096 | 77.85 | 640 | 0 | 8.22 | 8418.72 | 1664.78 | 10083.50 | 154.28 | 214.19 | 756.53 | 16.85 | 18.82 | 19.49 | 3549.62 | 4954.17 | 5479.06 | ✅ |
| c32_i1024_o512 | 32 | 1024 | 512 | 77.73 | 640 | 0 | 8.23 | 8431.17 | 1700.51 | 10131.68 | 155.17 | 216.90 | 751.85 | 16.62 | 18.26 | 18.67 | 3574.12 | 4831.87 | 5478.82 | ✅ |
| c32_i16384_o1024 | 32 | 16384 | 1024 | 277.96 | 640 | 0 | 2.30 | 37723.90 | 563.76 | 38287.65 | 1266.27 | 1984.25 | 8777.02 | 48.75 | 60.64 | 65.85 | 13073.64 | 26643.75 | 35597.74 | ✅ |
| c32_i16384_o128 | 32 | 16384 | 128 | 240.94 | 640 | 0 | 2.66 | 43520.72 | 338.90 | 43859.62 | 3738.21 | 4848.25 | 8794.30 | 61.61 | 81.93 | 82.76 | 11539.01 | 13150.69 | 16925.01 | ❌ |
| c32_i16384_o2048 | 32 | 16384 | 2048 | 277.48 | 640 | 0 | 2.31 | 37789.10 | 576.94 | 38366.04 | 1268.99 | 2250.82 | 8796.37 | 47.57 | 58.28 | 64.83 | 13129.88 | 24681.93 | 38211.25 | ✅ |
| c32_i16384_o256 | 32 | 16384 | 256 | 264.70 | 640 | 0 | 2.42 | 39614.37 | 508.66 | 40123.04 | 1298.03 | 1994.66 | 8761.72 | 54.06 | 64.54 | 70.67 | 12625.00 | 16418.68 | 19908.81 | ⚠️ |
| c32_i16384_o512 | 32 | 16384 | 512 | 275.93 | 640 | 0 | 2.32 | 38001.51 | 561.14 | 38562.65 | 1268.67 | 1990.86 | 8787.03 | 48.89 | 60.15 | 66.01 | 13034.17 | 25370.34 | 27540.55 | ✅ |
| c32_i2048_o1024 | 32 | 2048 | 1024 | 87.95 | 640 | 0 | 7.28 | 14903.68 | 1574.85 | 16478.54 | 200.89 | 270.09 | 1352.10 | 17.77 | 19.91 | 20.51 | 4023.46 | 6506.86 | 8623.71 | ✅ |
| c32_i2048_o128 | 32 | 2048 | 128 | 59.24 | 640 | 0 | 10.80 | 22125.82 | 1375.94 | 23501.76 | 367.53 | 672.56 | 1352.00 | 18.71 | 21.55 | 22.09 | 2732.47 | 3094.92 | 3404.73 | ✅ |
| c32_i2048_o2048 | 32 | 2048 | 2048 | 89.13 | 640 | 0 | 7.18 | 14705.49 | 1556.10 | 16261.59 | 199.91 | 291.10 | 1361.13 | 17.87 | 20.28 | 20.98 | 4056.53 | 6825.74 | 9339.78 | ✅ |
| c32_i2048_o256 | 32 | 2048 | 256 | 83.40 | 640 | 0 | 7.67 | 15715.67 | 1544.12 | 17259.80 | 202.63 | 380.57 | 1355.98 | 18.19 | 20.18 | 20.73 | 3838.39 | 5110.33 | 5339.24 | ✅ |
| c32_i2048_o4096 | 32 | 2048 | 4096 | 88.69 | 640 | 0 | 7.22 | 14778.01 | 1520.45 | 16298.46 | 199.19 | 288.11 | 1360.76 | 18.06 | 20.41 | 20.84 | 3982.98 | 6383.30 | 9405.39 | ✅ |
| c32_i2048_o512 | 32 | 2048 | 512 | 90.84 | 640 | 0 | 7.05 | 14428.41 | 1530.32 | 15958.73 | 197.17 | 253.30 | 1349.46 | 17.88 | 20.25 | 21.02 | 4060.40 | 6700.48 | 9080.73 | ✅ |
| c32_i4096_o1024 | 32 | 4096 | 1024 | 113.81 | 640 | 0 | 5.62 | 23032.89 | 1212.69 | 24245.58 | 339.67 | 542.37 | 2422.62 | 22.72 | 26.10 | 27.21 | 5215.72 | 8904.34 | 13457.62 | ✅ |
| c32_i4096_o128 | 32 | 4096 | 128 | 85.66 | 640 | 0 | 7.47 | 30604.61 | 950.87 | 31555.48 | 556.18 | 1180.23 | 2449.13 | 27.29 | 31.52 | 34.82 | 4002.80 | 4613.66 | 5507.05 | ✅ |
| c32_i4096_o2048 | 32 | 4096 | 2048 | 114.27 | 640 | 0 | 5.60 | 22939.90 | 1189.49 | 24129.39 | 341.11 | 550.60 | 2435.46 | 22.78 | 26.02 | 27.73 | 5165.78 | 9243.12 | 13018.90 | ✅ |
| c32_i4096_o256 | 32 | 4096 | 256 | 106.21 | 640 | 0 | 6.03 | 24681.91 | 1178.61 | 25860.52 | 350.79 | 564.37 | 2447.99 | 23.55 | 26.71 | 28.32 | 4937.54 | 6734.16 | 7405.49 | ✅ |
| c32_i4096_o4096 | 32 | 4096 | 4096 | 114.94 | 640 | 0 | 5.57 | 22806.49 | 1192.22 | 23998.71 | 341.86 | 550.55 | 2448.39 | 22.83 | 26.16 | 27.23 | 5203.63 | 8545.93 | 12625.46 | ✅ |
| c32_i4096_o512 | 32 | 4096 | 512 | 112.20 | 640 | 0 | 5.70 | 23364.93 | 1200.40 | 24565.33 | 340.20 | 560.25 | 2385.06 | 22.99 | 26.29 | 27.73 | 5150.64 | 8529.09 | 12164.94 | ✅ |
| c32_i512_o256 | 32 | 512 | 256 | 78.27 | 640 | 0 | 8.18 | 4186.51 | 1690.88 | 5877.39 | 149.00 | 289.93 | 473.49 | 16.81 | 19.27 | 22.36 | 3610.39 | 4786.53 | 5755.87 | ✅ |
| c32_i65536_o1024 | 32 | 65536 | 1024 | 266.52 | 160 | 0 | 0.60 | 39343.10 | 131.31 | 39474.42 | 8656.40 | 33860.81 | 43057.06 | 171.14 | 244.27 | 281.95 | 46303.99 | 103438.83 | 127750.75 | ❌ |
| c4_i512_o256 | 4 | 512 | 256 | 43.40 | 80 | 0 | 1.84 | 943.85 | 379.18 | 1323.03 | 118.07 | 125.54 | 241.46 | 7.83 | 8.29 | 8.68 | 1720.27 | 2165.16 | 2253.32 | ✅ |
| c64_i512_o256 | 64 | 512 | 256 | 109.33 | 1280 | 0 | 11.71 | 5994.26 | 2402.36 | 8396.62 | 171.45 | 299.06 | 771.51 | 24.39 | 26.72 | 27.58 | 5151.60 | 6720.29 | 6878.42 | ✅ |
| c8_i1024_o1024 | 8 | 1024 | 1024 | 47.80 | 160 | 0 | 3.35 | 3427.67 | 696.10 | 4123.77 | 129.12 | 192.22 | 281.04 | 9.27 | 9.95 | 10.24 | 2052.01 | 2788.77 | 3052.14 | ✅ |
| c8_i1024_o128 | 8 | 1024 | 128 | 31.03 | 160 | 0 | 5.16 | 5279.38 | 659.92 | 5939.30 | 187.55 | 269.21 | 292.69 | 8.88 | 9.90 | 9.97 | 1315.49 | 1387.68 | 1458.94 | ✅ |
| c8_i1024_o2048 | 8 | 1024 | 2048 | 47.63 | 160 | 0 | 3.36 | 3439.84 | 684.76 | 4124.59 | 139.64 | 191.21 | 283.85 | 9.49 | 10.93 | 11.77 | 2064.28 | 2867.60 | 3177.60 | ✅ |
| c8_i1024_o256 | 8 | 1024 | 256 | 47.68 | 160 | 0 | 3.36 | 3436.34 | 684.94 | 4121.28 | 134.22 | 195.34 | 290.76 | 9.44 | 10.83 | 11.66 | 2050.72 | 2669.20 | 2870.35 | ✅ |
| c8_i1024_o4096 | 8 | 1024 | 4096 | 48.02 | 160 | 0 | 3.33 | 3411.81 | 683.42 | 4095.23 | 132.59 | 194.92 | 285.39 | 9.39 | 10.45 | 11.05 | 2050.12 | 2647.51 | 2950.32 | ✅ |
| c8_i1024_o512 | 8 | 1024 | 512 | 48.89 | 160 | 0 | 3.27 | 3351.35 | 699.85 | 4051.20 | 128.43 | 130.82 | 286.17 | 9.25 | 9.96 | 10.23 | 2097.31 | 2846.33 | 3059.68 | ✅ |
| c8_i131072_o512 | 8 | 131072 | 512 | 593.71 | 160 | 0 | 0.27 | 35322.95 | 49.97 | 35372.92 | 8837.26 | 12856.65 | 21274.84 | 95.98 | 174.74 | 250.31 | 26223.81 | 43791.96 | 52427.10 | ❌ |
| c8_i16384_o1024 | 8 | 16384 | 1024 | 117.03 | 160 | 0 | 1.37 | 22399.35 | 381.47 | 22780.82 | 880.55 | 1325.49 | 2519.24 | 15.14 | 18.88 | 22.89 | 5043.96 | 9664.45 | 11942.89 | ✅ |
| c8_i16384_o128 | 8 | 16384 | 128 | 83.84 | 160 | 0 | 1.91 | 31265.56 | 241.22 | 31506.78 | 1766.89 | 2637.47 | 2676.54 | 14.95 | 23.25 | 26.08 | 3640.27 | 3713.62 | 5332.84 | ✅ |
| c8_i16384_o2048 | 8 | 16384 | 2048 | 111.65 | 160 | 0 | 1.43 | 23479.38 | 353.00 | 23832.38 | 858.33 | 1105.07 | 2515.80 | 15.80 | 19.89 | 21.08 | 4778.32 | 8910.12 | 13501.67 | ✅ |
| c8_i16384_o256 | 8 | 16384 | 256 | 101.77 | 160 | 0 | 1.57 | 25759.71 | 329.17 | 26088.88 | 867.77 | 1267.33 | 2511.92 | 17.06 | 21.93 | 23.13 | 4427.86 | 5958.68 | 6667.56 | ✅ |
| c8_i16384_o512 | 8 | 16384 | 512 | 109.64 | 160 | 0 | 1.46 | 23908.84 | 355.69 | 24264.53 | 908.35 | 1361.76 | 2515.33 | 15.82 | 20.35 | 22.15 | 4755.66 | 8684.67 | 10122.99 | ✅ |
| c8_i2048_o1024 | 8 | 2048 | 1024 | 53.11 | 160 | 0 | 3.01 | 6170.36 | 677.84 | 6848.20 | 150.15 | 153.04 | 410.55 | 9.49 | 10.24 | 10.65 | 2286.09 | 3870.04 | 5178.54 | ✅ |
| c8_i2048_o128 | 8 | 2048 | 128 | 33.35 | 160 | 0 | 4.80 | 9826.22 | 612.40 | 10438.62 | 228.93 | 388.77 | 404.46 | 9.33 | 10.33 | 11.02 | 1411.23 | 1530.95 | 1595.88 | ✅ |
| c8_i2048_o2048 | 8 | 2048 | 2048 | 51.12 | 160 | 0 | 3.13 | 6409.90 | 672.70 | 7082.60 | 150.70 | 165.44 | 418.00 | 9.56 | 10.25 | 10.68 | 2202.49 | 3406.00 | 4277.62 | ✅ |
| c8_i2048_o256 | 8 | 2048 | 256 | 48.03 | 160 | 0 | 3.33 | 6822.11 | 658.85 | 7480.96 | 148.57 | 147.18 | 403.31 | 9.66 | 10.48 | 10.98 | 2051.77 | 2715.99 | 2745.12 | ✅ |
| c8_i2048_o4096 | 8 | 2048 | 4096 | 53.12 | 160 | 0 | 3.01 | 6168.50 | 661.05 | 6829.55 | 150.99 | 219.64 | 421.18 | 9.58 | 10.44 | 10.58 | 2250.24 | 3788.73 | 4649.51 | ✅ |
| c8_i2048_o512 | 8 | 2048 | 512 | 50.97 | 160 | 0 | 3.14 | 6428.69 | 669.04 | 7097.73 | 148.41 | 147.71 | 403.41 | 9.60 | 10.48 | 11.08 | 2190.12 | 3392.37 | 4269.13 | ✅ |
| c8_i262144_o128 | 8 | 262144 | 128 | 1333.56 | 160 | 0 | 0.12 | 31451.94 | 12.61 | 31464.55 | 36128.45 | 57997.25 | 58592.27 | 215.95 | 392.14 | 446.94 | 59143.65 | 61777.27 | 74358.75 | ❌ |
| c8_i4096_o1024 | 8 | 4096 | 1024 | 60.18 | 160 | 0 | 2.66 | 10890.64 | 578.38 | 11469.02 | 253.10 | 364.29 | 731.93 | 10.57 | 11.89 | 12.17 | 2550.26 | 4642.49 | 7535.89 | ✅ |
| c8_i4096_o128 | 8 | 4096 | 128 | 40.35 | 160 | 0 | 3.96 | 16240.24 | 506.02 | 16746.26 | 376.51 | 695.40 | 727.26 | 10.62 | 11.99 | 13.73 | 1721.35 | 1910.10 | 2181.08 | ✅ |
| c8_i4096_o2048 | 8 | 4096 | 2048 | 57.14 | 160 | 0 | 2.80 | 11470.28 | 593.22 | 12063.50 | 254.35 | 361.27 | 732.14 | 10.52 | 11.68 | 12.36 | 2478.80 | 3843.55 | 5407.45 | ✅ |
| c8_i4096_o256 | 8 | 4096 | 256 | 54.73 | 160 | 0 | 2.92 | 11973.53 | 575.24 | 12548.76 | 252.76 | 366.29 | 732.21 | 10.63 | 12.02 | 12.59 | 2342.56 | 3122.99 | 3247.98 | ✅ |
| c8_i4096_o4096 | 8 | 4096 | 4096 | 61.97 | 160 | 0 | 2.58 | 10574.93 | 602.60 | 11177.53 | 251.62 | 276.31 | 733.83 | 10.48 | 11.81 | 12.23 | 2715.50 | 3797.26 | 5487.12 | ✅ |
| c8_i4096_o512 | 8 | 4096 | 512 | 57.59 | 160 | 0 | 2.78 | 11379.47 | 586.75 | 11966.22 | 255.02 | 376.23 | 733.95 | 10.54 | 11.70 | 12.17 | 2482.90 | 4048.32 | 5636.75 | ✅ |
| c8_i512_o256 | 8 | 512 | 256 | 48.34 | 160 | 0 | 3.31 | 1694.53 | 687.00 | 2381.52 | 125.68 | 194.69 | 258.22 | 9.43 | 10.43 | 10.95 | 2075.15 | 2669.68 | 2871.40 | ✅ |
| c8_i65536_o1024 | 8 | 65536 | 1024 | 298.66 | 160 | 0 | 0.54 | 35109.32 | 107.56 | 35216.88 | 3847.52 | 5884.53 | 9844.52 | 46.16 | 82.22 | 101.31 | 13001.39 | 21336.69 | 30533.51 | ⚠️ |
| c8_i65536_o128 | 8 | 65536 | 128 | 274.11 | 160 | 0 | 0.58 | 38254.29 | 70.30 | 38324.59 | 7223.66 | 11101.75 | 11262.28 | 40.50 | 72.54 | 72.63 | 12089.84 | 12347.21 | 12881.32 | ⚠️ |
| c8_i65536_o256 | 8 | 65536 | 256 | 291.01 | 160 | 0 | 0.55 | 36032.88 | 98.87 | 36131.75 | 3804.89 | 5718.15 | 9852.36 | 50.92 | 78.36 | 96.49 | 12843.24 | 18584.33 | 21293.29 | ❌ |
| c8_i65536_o512 | 8 | 65536 | 512 | 297.04 | 160 | 0 | 0.54 | 35301.20 | 111.90 | 35413.10 | 3902.29 | 6055.87 | 9835.60 | 44.23 | 73.93 | 83.82 | 13139.03 | 22983.14 | 31590.46 | ⚠️ |
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.

6429
experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json Normal file
View File

File diff suppressed because it is too large Load Diff

126
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md
View File

@ -1,126 +0,0 @@
# dsv4_h200_vllm_tp4_custom_bench_no_fp8
## 目的
当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU：
> **注意**：本实验变体 **不启用 `--kv-cache-dtype fp8`**，用于对比 FP8 KV Cache 与默认精度的性能差异。
| 服务 | 端口 | GPU 组 |
|------|------|--------|
| 1 | 30005 | 0, 1, 2, 3 |
| 2 | 30006 | 4, 5, 6, 7 |
相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。
## 与 TP=2 方案的区别
| 特性 | TP=2 (4 服务) | TP=4 (2 服务) |
|------|---------------|---------------|
| 服务数量 | 4 | 2 |
| 每服务 GPU | 2 | 4 |
| 单服务吞吐 | 较低 | 较高 |
| 集群总吞吐 | 相近 | 相近 |
| 适用场景 | 低并发、低延迟 | 高并发、大 batch |
## 文件说明
| 文件 | 说明 |
|------|------|
| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） |
| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） |
| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 |
| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） |
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
## 快速运行
```bash
# 完整流程（自动启动 2 个服务、压测、生成报告）
bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh
# 跳过服务管理（已有 2 个服务在运行）
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh
# 单独运行压测客户端（多服务，带健康检查）
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \
--host 127.0.0.1 \
--ports 30005,30006 \
--model deepseek-v4-flash \
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
--lb-strategy round_robin \
--health-check-interval 5 \
--health-max-failures 2 \
--health-recovery-interval 10 \
--request-max-retries 2 \
--output-file /tmp/test.jsonl
# 单独运行压测客户端（单服务，向后兼容）
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \
--output-file /tmp/test_single.jsonl
# 解析已有结果
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \
experiments/dsv4_h200_vllm_tp4_custom_bench/results/<RUN_ID>
```
## 场景设计
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|------|----------|----------|--------|------|
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req |
| 4 | 512 | 256 | 80 | 中低并发 |
| 8 | 512 | 256 | 160 | 中等并发 |
| 16 | 512 | 256 | 320 | 中高并发 |
| 32 | 512 | 256 | 640 | 高并发 |
| 64 | 512 | 256 | 1280 | 很高并发 |
| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 |
| 1 | 2048 | 512 | 20 | 长输入基线 |
| 8 | 2048 | 512 | 160 | 长输入中并发 |
| 32 | 2048 | 512 | 640 | 长输入高并发 |
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
## 健康检查与故障自动跳过
同 TP=2 版本，详见原 README。
## 负载均衡策略
同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。
## 输出格式
同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。
## 单独启动/停止 2 个服务
```bash
# 启动 2 个服务
bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh
# 停止（run_bench.sh 会自动停止，但也可以手动）
for port in 30005 30006; do
pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid"
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
rm -f "$pid_file"
done
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
```
## 注意事项
- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。
- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。
- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。
- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。
- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。

617
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py
View File

@ -1,617 +0,0 @@
#!/usr/bin/env python3
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
Sends concurrent requests to multiple inference servers and records per-request
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
Designed for TP=2 on 8-GPU setups where 4 independent services run on
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
round-robin or random load balancing. Unhealthy services are automatically
skipped with periodic retry.
Usage:
# Single service (backward compatible)
python3 bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--output-file results.jsonl
# Multi-service (4 services on ports 30005-30008)
python3 bench_client.py \
--host 127.0.0.1 \
--ports 30005,30006,30007,30008 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--lb-strategy round_robin \
--output-file results.jsonl
The output JSONL contains one object per request with raw timing data.
A final summary line ("completed" field) aggregates all requests.
"""
import argparse
import asyncio
import json
import random
import sys
import time
import urllib.request
from dataclasses import asdict, dataclass, field
from typing import Any
# Try aiohttp first; fall back to urllib for stdlib-only environments.
try:
import aiohttp
HAS_AIOHTTP = True
except ImportError:
HAS_AIOHTTP = False
# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------
@dataclass
class RequestResult:
request_id: int
input_tokens: int = 0
output_tokens: int = 0
first_token_time_ms: float = 0.0 # TTFT
e2e_latency_ms: float = 0.0 # total wall time
inter_token_latencies: list[float] = field(default_factory=list)
success: bool = False
error: str = ""
target_port: int = 0 # which service handled this request
retry_count: int = 0 # how many times this request was retried
@property
def tpot_ms(self) -> float:
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
if self.output_tokens <= 1:
return 0.0
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
@property
def mean_itl_ms(self) -> float:
if not self.inter_token_latencies:
return 0.0
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
def to_dict(self) -> dict[str, Any]:
d = asdict(self)
d["tpot_ms"] = self.tpot_ms
d["mean_itl_ms"] = self.mean_itl_ms
return d
# ---------------------------------------------------------------------------
# Health checker
# ---------------------------------------------------------------------------
class HealthChecker:
"""Periodically check service health and maintain a healthy-port list."""
def __init__(
self,
host: str,
ports: list[int],
check_interval: float = 5.0,
max_failures: int = 2,
recovery_interval: float = 10.0,
):
self.host = host
self.all_ports = ports
self.check_interval = check_interval
self.max_failures = max_failures
self.recovery_interval = recovery_interval
# port -> consecutive failure count
self._failures: dict[int, int] = {p: 0 for p in ports}
# port -> last healthy timestamp
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
self._lock = asyncio.Lock()
self._task: asyncio.Task | None = None
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
url = f"http://{self.host}:{port}/health"
try:
if HAS_AIOHTTP and session is not None:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
return resp.status == 200
else:
req = urllib.request.Request(url, method="GET")
with urllib.request.urlopen(req, timeout=5) as resp:
return resp.status == 200
except Exception:
return False
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
checks = [self._check_one(p, session) for p in self.all_ports]
results = await asyncio.gather(*checks, return_exceptions=True)
now = time.monotonic()
async with self._lock:
for port, ok in zip(self.all_ports, results):
if isinstance(ok, Exception):
ok = False
if ok:
self._failures[port] = 0
self._last_healthy[port] = now
else:
self._failures[port] += 1
async def start(self, session: aiohttp.ClientSession | None) -> None:
async def _loop() -> None:
while True:
await asyncio.sleep(self.check_interval)
await self._check_all(session)
self._task = asyncio.create_task(_loop())
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
@property
async def healthy_ports(self) -> list[int]:
now = time.monotonic()
async with self._lock:
healthy = []
for p in self.all_ports:
if self._failures[p] < self.max_failures:
healthy.append(p)
elif now - self._last_healthy[p] > self.recovery_interval:
# Give it another chance after recovery_interval.
self._failures[p] = max(0, self._failures[p] - 1)
healthy.append(p)
return healthy
async def is_healthy(self, port: int) -> bool:
ports = await self.healthy_ports
return port in ports
# ---------------------------------------------------------------------------
# Load balancer with failover
# ---------------------------------------------------------------------------
class LoadBalancer:
"""Distribute requests across healthy backend ports with auto-failover."""
def __init__(
self,
ports: list[int],
health_checker: HealthChecker,
strategy: str = "round_robin",
):
self.all_ports = ports
self.health_checker = health_checker
self.strategy = strategy
self._idx = 0
self._lock = asyncio.Lock()
def _next_round_robin(self, healthy: list[int]) -> int:
if not healthy:
return self.all_ports[self._idx % len(self.all_ports)]
for _ in range(len(self.all_ports)):
port = self.all_ports[self._idx % len(self.all_ports)]
self._idx += 1
if port in healthy:
return port
# Fallback: all unhealthy, pick first healthy anyway.
return healthy[0] if healthy else self.all_ports[0]
def _next_random(self, healthy: list[int]) -> int:
if healthy:
return random.choice(healthy)
return random.choice(self.all_ports)
async def next_port(self) -> int:
healthy = await self.health_checker.healthy_ports
async with self._lock:
if self.strategy == "round_robin":
return self._next_round_robin(healthy)
elif self.strategy in ("random", "least_conn"):
return self._next_random(healthy)
else:
return self._next_round_robin(healthy)
@property
def port_count(self) -> int:
return len(self.all_ports)
# ---------------------------------------------------------------------------
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
tokens.extend(words)
return " ".join(tokens[:needed])
# ---------------------------------------------------------------------------
# aiohttp-based async request (preferred)
# ---------------------------------------------------------------------------
async def send_request_aiohttp(
session: aiohttp.ClientSession,
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
headers = {"Content-Type": "application/json"}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
first_token_received = False
last_token_time = 0.0
try:
async with session.post(url, json=payload, headers=headers) as resp:
if resp.status != 200:
text = await resp.text()
result.error = f"HTTP {resp.status}: {text[:200]}"
return result
async for line in resp.content:
line = line.decode("utf-8").strip()
if not line or not line.startswith("data: "):
continue
data_str = line[len("data: "):]
if data_str == "[DONE]":
break
try:
chunk = json.loads(data_str)
except json.JSONDecodeError:
continue
choices = chunk.get("choices", [])
if not choices:
continue
delta = choices[0].get("delta", {})
content = delta.get("content", "")
if content:
now = time.perf_counter()
if not first_token_received:
result.first_token_time_ms = (now - start_time) * 1000
first_token_received = True
else:
result.inter_token_latencies.append((now - last_token_time) * 1000)
last_token_time = now
result.output_tokens += 1
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.input_tokens = len(prompt.split()) # approximate
result.success = True
except asyncio.TimeoutError:
result.error = "timeout"
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# urllib-based synchronous request (fallback, no aiohttp)
# ---------------------------------------------------------------------------
def send_request_urllib(
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": False,
}
data = json.dumps(payload).encode("utf-8")
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
try:
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=600) as resp:
body = json.loads(resp.read().decode("utf-8"))
choices = body.get("choices", [])
if not choices:
result.error = "no choices in response"
return result
content = choices[0].get("message", {}).get("content", "")
result.output_tokens = len(content.split()) if content else 0
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.first_token_time_ms = result.e2e_latency_ms
result.input_tokens = len(prompt.split())
result.success = True
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# Unified send_request wrapper with retry & failover
# ---------------------------------------------------------------------------
async def send_request_with_retry(
session: aiohttp.ClientSession | None,
host: str,
lb: LoadBalancer,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
max_retries: int = 2,
) -> RequestResult:
"""Send a request, retrying on failure with a different port each time."""
result = RequestResult(request_id=request_id)
for attempt in range(max_retries + 1):
port = await lb.next_port()
if HAS_AIOHTTP and session is not None:
result = await send_request_aiohttp(
session, host, port, model, request_id, prompt, max_tokens, temperature
)
else:
loop = asyncio.get_event_loop()
result = await loop.run_in_executor(
None, send_request_urllib,
host, port, model, request_id, prompt, max_tokens, temperature
)
result.retry_count = attempt
if result.success:
return result
# If failed, try another port on next iteration (unless last attempt).
if attempt < max_retries:
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
return result
# ---------------------------------------------------------------------------
# Benchmark orchestration
# ---------------------------------------------------------------------------
async def run_benchmark(
host: str,
ports: list[int],
lb: LoadBalancer,
health_checker: HealthChecker,
model: str,
concurrency: int,
input_len: int,
output_len: int,
num_prompts: int,
temperature: float,
output_file: str,
) -> dict[str, Any]:
"""Run the benchmark and write a JSONL file."""
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
results: list[RequestResult] = []
semaphore = asyncio.Semaphore(concurrency)
start = time.perf_counter()
if HAS_AIOHTTP:
connector = aiohttp.TCPConnector(limit=concurrency * 2)
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
await health_checker.start(session)
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
else:
await health_checker.start(None)
from concurrent.futures import ThreadPoolExecutor
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
None, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
duration = time.perf_counter() - start
# Write JSONL
with open(output_file, "w", encoding="utf-8") as f:
for r in results:
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
# Compute summary
successful = [r for r in results if r.success]
failed = len(results) - len(successful)
retried = [r for r in results if r.retry_count > 0]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
ttfts = [r.first_token_time_ms for r in successful]
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
e2es = [r.e2e_latency_ms for r in successful]
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
total_input_tokens = sum(r.input_tokens for r in successful)
total_output_tokens = sum(r.output_tokens for r in successful)
# Per-port breakdown
per_port_stats: dict[int, dict[str, Any]] = {}
for port in ports:
port_results = [r for r in successful if r.target_port == port]
port_failed = [r for r in results if r.target_port == port and not r.success]
if port_results or port_failed:
port_ttfts = [r.first_token_time_ms for r in port_results]
port_e2es = [r.e2e_latency_ms for r in port_results]
per_port_stats[port] = {
"count": len(port_results),
"failed": len(port_failed),
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
}
# Health-check stats
healthy_at_end = await health_checker.healthy_ports
summary = {
"completed": len(successful),
"failed": failed,
"retried": len(retried),
"duration": duration,
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
"total_input_tokens": total_input_tokens,
"total_output_tokens": total_output_tokens,
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
"median_ttft_ms": percentile(ttfts, 50),
"p90_ttft_ms": percentile(ttfts, 90),
"p95_ttft_ms": percentile(ttfts, 95),
"p99_ttft_ms": percentile(ttfts, 99),
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
"median_tpot_ms": percentile(tpots, 50),
"p90_tpot_ms": percentile(tpots, 90),
"p95_tpot_ms": percentile(tpots, 95),
"p99_tpot_ms": percentile(tpots, 99),
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
"median_e2e_latency_ms": percentile(e2es, 50),
"p90_e2e_latency_ms": percentile(e2es, 90),
"p95_e2e_latency_ms": percentile(e2es, 95),
"p99_e2e_latency_ms": percentile(e2es, 99),
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
"median_itl_ms": percentile(itls, 50),
"p90_itl_ms": percentile(itls, 90),
"p95_itl_ms": percentile(itls, 95),
"p99_itl_ms": percentile(itls, 99),
"per_port_stats": per_port_stats,
"healthy_ports_at_end": healthy_at_end,
"lb_strategy": lb.strategy,
"num_services": lb.port_count,
}
# Append summary as the last line
with open(output_file, "a", encoding="utf-8") as f:
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
return summary
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=30005,
help="Single service port (backward compatible)")
parser.add_argument("--ports", default="",
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
parser.add_argument("--model", default="deepseek-v4-flash")
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--input-len", type=int, default=512)
parser.add_argument("--output-len", type=int, default=256)
parser.add_argument("--num-prompts", type=int, default=10)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--lb-strategy", default="round_robin",
choices=["round_robin", "random", "least_conn"],
help="Load balancing strategy across services")
parser.add_argument("--health-check-interval", type=float, default=5.0,
help="Seconds between health checks (default: 5)")
parser.add_argument("--health-max-failures", type=int, default=2,
help="Consecutive failures before marking a port unhealthy (default: 2)")
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
help="Seconds before retrying an unhealthy port (default: 10)")
parser.add_argument("--request-max-retries", type=int, default=2,
help="Max retries per request on failure (default: 2)")
parser.add_argument("--output-file", required=True)
args = parser.parse_args()
# Determine ports: --ports takes precedence, else fall back to --port.
if args.ports:
ports = [int(p.strip()) for p in args.ports.split(",")]
else:
ports = [args.port]
health_checker = HealthChecker(
host=args.host,
ports=ports,
check_interval=args.health_check_interval,
max_failures=args.health_max_failures,
recovery_interval=args.health_recovery_interval,
)
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
summary = asyncio.run(run_benchmark(
host=args.host,
ports=ports,
lb=lb,
health_checker=health_checker,
model=args.model,
concurrency=args.concurrency,
input_len=args.input_len,
output_len=args.output_len,
num_prompts=args.num_prompts,
temperature=args.temperature,
output_file=args.output_file,
))
print(json.dumps(summary, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()

84
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env
View File

@ -1,84 +0,0 @@
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
#
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
# Service 1: port 30005, GPUs 0,1
# Service 2: port 30006, GPUs 2,3
# Service 3: port 30007, GPUs 4,5
# Service 4: port 30008, GPUs 6,7
#
# bench_client.py distributes requests across all 4 services via round-robin
# load balancing, giving accurate cluster-wide throughput even at low
# per-service concurrency.
EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench_no_fp8"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
# Primary port (backward compatible); multi-service ports are in PORTS.
PORT="30005"
PORTS="30005,30006"
BACKEND="vllm"
ENGINE="vllm"
# Native virtual environments on the host.
VENV_SERVER="/data/user1/yy/envs/vllm"
VENV_CLIENT="/data/user1/yy/envs/vllm"
# Load balancing strategy: round_robin | random | least_conn
LB_STRATEGY="round_robin"
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
# For multi-service we include low-concurrency scenarios to show how
# requests spread across 2 services.
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
)
# Server start script bundled with this experiment.
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

206
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py
View File

@ -1,206 +0,0 @@
#!/usr/bin/env python3
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
Reads JSONL files produced by bench_client.py (one request per line,
last line is the summary) and generates:
- results.json (appended scenarios, following the standard schema)
- report.md (human-readable summary)
Usage:
python3 parse_results.py <result_root>
"""
import json
import sys
from pathlib import Path
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
"""Read the summary JSON (last line) and all per-request lines."""
requests: list[dict] = []
summary: dict | None = None
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
# The summary line has "completed" and "duration" keys.
if "completed" in obj and "duration" in obj:
summary = obj
else:
requests.append(obj)
return summary, requests
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
completed = summary.get("completed", 0)
total = len(requests)
failed = total - completed if total > 0 else 0
duration_s = summary.get("duration", 0.0)
# Extract per-request metrics for percentiles.
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
return {
"success": completed,
"failed": failed,
"duration_s": duration_s,
"request_throughput": summary.get("request_throughput", 0.0),
"input_token_throughput": summary.get("input_throughput", 0.0),
"output_token_throughput": summary.get("output_throughput", 0.0),
"total_token_throughput": summary.get("total_throughput", 0.0),
"total_input_tokens": summary.get("total_input_tokens", 0),
"total_output_tokens": summary.get("total_output_tokens", 0),
"e2e_ms": {
"mean": summary.get("mean_e2e_latency_ms", 0.0),
"p50": percentile(e2es, 50),
"p90": percentile(e2es, 90),
"p95": percentile(e2es, 95),
"p99": percentile(e2es, 99),
},
"ttft_ms": {
"mean": summary.get("mean_ttft_ms", 0.0),
"p50": percentile(ttfts, 50),
"p90": percentile(ttfts, 90),
"p95": percentile(ttfts, 95),
"p99": percentile(ttfts, 99),
},
"tpot_ms": {
"mean": summary.get("mean_tpot_ms", 0.0),
"p50": percentile(tpots, 50),
"p90": percentile(tpots, 90),
"p95": percentile(tpots, 95),
"p99": percentile(tpots, 99),
},
"itl_ms": {
"mean": summary.get("mean_itl_ms", 0.0),
"p50": percentile(itls, 50),
"p90": percentile(itls, 90),
"p95": percentile(itls, 95),
"p99": percentile(itls, 99),
},
}
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
return f"c{concurrency}_i{input_len}_o{output_len}"
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
if ttft_ok and tpot_ok:
mark = "✅"
elif ttft_ok or tpot_ok:
mark = "⚠️"
else:
mark = "❌"
return {
"ttft_p95_ok": ttft_ok,
"tpot_mean_ok": tpot_ok,
"overall": mark,
}
def append_scenario(results_json: Path, scenario: dict) -> None:
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"].append(scenario)
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
report_path = result_root / "report.md"
with open(report_path, "w", encoding="utf-8") as f:
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
f.write(f"- **Result root**: `{result_root}`\n\n")
f.write("## Results\n\n")
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
for s in scenarios:
cfg = s["config"]
m = s["metrics"]
slo = s.get("slo_status", {}).get("overall", "")
f.write(
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['total_token_throughput']:.2f} | "
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
)
f.write("\n")
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
def main() -> None:
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
raw_dir = result_root / "raw_outputs"
results_json = result_root / "results.json"
if not raw_dir.exists():
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
scenarios = []
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
parts = jsonl_path.stem.split("_")
if len(parts) < 5:
continue
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
summary, requests = parse_jsonl(jsonl_path)
if summary is None:
continue
metrics = compute_metrics(summary, requests)
scenario = {
"name": scenario_name(concurrency, input_len, output_len),
"config": {
"concurrency": concurrency,
"input_len": input_len,
"output_len": output_len,
"dataset": "random",
"num_prompts": metrics["success"] + metrics["failed"],
},
"metrics": metrics,
"slo_status": slo_status(metrics),
"raw_file": str(jsonl_path),
}
scenarios.append(scenario)
if not scenarios:
print("No benchmark outputs found to parse")
return
if results_json.exists():
for s in scenarios:
append_scenario(results_json, s)
generate_report(result_root, scenarios)
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
if __name__ == "__main__":
main()

278
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh
View File

@ -1,278 +0,0 @@
#!/usr/bin/env bash
# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200.
#
# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7).
# bench_client.py distributes requests across all 2 services via round-robin LB,
# giving accurate cluster-wide throughput even at low per-service concurrency.
#
# Usage:
# bash run_bench.sh
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
RAW_DIR="${RESULT_ROOT}/raw_outputs"
LOG_DIR="${RESULT_ROOT}/logs"
ensure_result_root "$RESULT_ROOT"
log_init "${LOG_DIR}/orchestrator.log"
log "experiment=${EXPERIMENT_NAME}"
log "run_id=${RUN_ID}"
log "result_root=${RESULT_ROOT}"
log "platform=${PLATFORM}"
log "chip=${CHIP}"
log "accelerator=${ACCELERATOR}"
log "engine=${ENGINE}"
log "hardware=${HARDWARE}"
log "model=${MODEL_PATH}"
log "services=${PORTS}"
log "lb_strategy=${LB_STRATEGY}"
log "server_start_script=${SERVER_START_SCRIPT}"
# Write initial metadata for this run.
METADATA_JSON="${RESULT_ROOT}/results.json"
write_metadata_json \
"$METADATA_JSON" \
"$EXPERIMENT_NAME" \
"$RUN_ID" \
"$MODEL_PATH" \
"$BACKEND" \
"$ENGINE" \
"$HARDWARE" \
"$ACCELERATOR" \
"$CHIP" \
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
"$VENV_SERVER" \
"H200 2x vLLM TP=4 multi-service benchmark using bench_client.py (no FP8 KV cache)"
# Update metadata with config.
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
import json, sys
path = sys.argv[1]
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
data["config"] = {
"tp": 4,
"num_services": 2,
"ports": "30005,30006",
"gpu_groups": "0,1,2,3|4,5,6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20", "2 512 256 40", "4 512 256 80",
"8 512 256 160", "16 512 256 320", "32 512 256 640",
"64 512 256 1280", "128 512 256 2560",
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
PY
# ---------------------------------------------------------------------------
# Server helpers (multi-service)
# ---------------------------------------------------------------------------
is_server_healthy() {
local port="$1"
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
}
are_all_services_healthy() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
for p in "${PORT_LIST[@]}"; do
if ! is_server_healthy "$p"; then
return 1
fi
done
return 0
}
stop_all_servers() {
log "stopping all vllm services"
# Kill by PID files first.
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
[[ -f "$pid_file" ]] || continue
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "killing pid=${pid}"
kill "$pid" 2>/dev/null || true
sleep 2
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
done
# Fallback: kill any remaining vllm processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
}
start_all_servers() {
log "starting all vllm services with ${SERVER_START_SCRIPT}"
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
exit 1
fi
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
if ! are_all_services_healthy "$PORTS"; then
log "error: not all vllm services became healthy"
exit 1
fi
log "all vllm services are healthy"
}
# ---------------------------------------------------------------------------
# Pre-benchmark health check with detailed reporting
# ---------------------------------------------------------------------------
check_services_detailed() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy_count=0
local unhealthy_ports=""
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
((healthy_count++))
else
unhealthy_ports="${unhealthy_ports}${p} "
fi
done
echo "$healthy_count"
if [[ -n "$unhealthy_ports" ]]; then
echo "unhealthy: ${unhealthy_ports}" >&2
fi
}
wait_for_all_services() {
local ports="$1"
local max_wait="${2:-240}"
local interval="${3:-5}"
local elapsed=0
while (( elapsed < max_wait )); do
local healthy_count
healthy_count="$(check_services_detailed "$ports" | head -1)"
IFS=',' read -ra PORT_LIST <<< "$ports"
if (( healthy_count == ${#PORT_LIST[@]} )); then
return 0
fi
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
sleep "$interval"
((elapsed += interval))
done
return 1
}
# ---------------------------------------------------------------------------
# Mid-benchmark health check (called between scenarios)
# ---------------------------------------------------------------------------
check_and_report_services() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy=()
local unhealthy=()
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
healthy+=("$p")
else
unhealthy+=("$p")
fi
done
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
if [[ ${#unhealthy[@]} -gt 0 ]]; then
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
fi
return ${#unhealthy[@]}
}
on_exit() {
local code=$?
log "orchestrator exiting with code=${code}"
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
stop_all_servers
fi
exit "$code"
}
trap on_exit EXIT
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
if ! wait_for_all_services "$PORTS"; then
log "error: not all healthy services found on ports ${PORTS}"
exit 1
fi
else
stop_all_servers
start_all_servers
fi
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
--host 127.0.0.1 \
--ports "$PORTS" \
--model "$SERVED_MODEL_NAME" \
--concurrency "$concurrency" \
--input-len "$input_len" \
--output-len "$output_len" \
--num-prompts "$num_prompts" \
--lb-strategy "$LB_STRATEGY" \
--health-check-interval 5 \
--health-max-failures 2 \
--health-recovery-interval 10 \
--request-max-retries 2 \
--output-file "$output_file" \
> "$detail_log" 2>&1 || {
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
# After failure, check service health before continuing.
check_and_report_services "$PORTS"
continue
}
log "finished scenario: output=${output_file}"
# Between scenarios, report service health so we can spot degradation early.
check_and_report_services "$PORTS" || true
done
log "===== BENCHMARK DONE ====="
log "parsing results"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
}
log "all results saved to ${RESULT_ROOT}"

107
experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh
View File

@ -1,107 +0,0 @@
#!/bin/bash
# Start 2 independent vLLM TP=4 services on 8x H200.
# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7).
# This maximizes per-service throughput for higher-concurrency workloads.
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
VENV="${VENV_SERVER}"
export PATH="$VENV/bin:$PATH"
VLLM="$VENV/bin/vllm"
TP=4
LOG_DIR="${SCRIPT_DIR}/logs"
mkdir -p "$LOG_DIR"
# 2 services, each on 4 GPUs.
# Port and GPU mapping.
SERVICES=(
"30005:0,1,2,3"
"30006:4,5,6,7"
)
# Kill any existing vLLM processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
# Remove old PID files.
rm -f "${SCRIPT_DIR}"/*.pid
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
GPUS="${svc##*:}"
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
echo "Log: ${LOG}"
CUDA_VISIBLE_DEVICES="${GPUS}" \
nohup "$VLLM" serve "$MODEL_PATH" \
--trust-remote-code \
--tensor-parallel-size "$TP" \
--block-size 256 \
--served-model-name "$SERVED_MODEL_NAME" \
--port "$PORT" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
done
echo ""
echo "Waiting for all 2 services to become healthy..."
MAX_WAIT=240
all_healthy=0
for i in $(seq 1 $MAX_WAIT); do
all_healthy=1
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
all_healthy=0
break
fi
done
if [[ "$all_healthy" -eq 1 ]]; then
echo "All 2 services are healthy!"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
echo " - http://127.0.0.1:${PORT}"
done
exit 0
fi
# Check if any process died early.
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
if [[ -f "$PID_FILE" ]]; then
PID="$(cat "$PID_FILE")"
if ! kill -0 "$PID" 2>/dev/null; then
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
tail -200 $LOG 2>/dev/null || true
exit 1
fi
fi
done
if (( i % 10 == 0 )); then
echo "Waiting... (${i}/${MAX_WAIT})"
fi
sleep 5
done
echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
echo "--- Last 50 lines of port ${PORT} ---"
tail -50 $LOG 2>/dev/null || true
done
exit 1

53
experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py
View File

@ -228,16 +228,22 @@ class LoadBalancer:
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
import uuid
def make_prompt(token_len: int) -> str:
prefix = str(uuid.uuid4())
words = [
"the", "quick", "brown", "fox",
"jumps", "over", "lazy", "dog",
"benchmark", "performance", "latency"
]
tokens = [prefix]
while len(tokens) < token_len:
tokens.extend(words)
return " ".join(tokens[:needed])
return " ".join(tokens[:token_len])
# ---------------------------------------------------------------------------
@ -440,6 +446,35 @@ async def run_benchmark(
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
print("Starting warmup...")
warmup_per_service = 10
warmup_requests = warmup_per_service * len(ports)
warmup_prompts = [
make_prompt(input_len)
for _ in range(warmup_requests)
]
warmup_tasks = [
send_request_with_retry(
session=session,
host=host,
lb=lb,
model=model,
request_id=-1,
prompt=p,
max_tokens=output_len,
temperature=temperature,
)
for p in warmup_prompts
]
await asyncio.gather(*warmup_tasks)
print("Warmup finished.")
# 给 CUDA / Scheduler 一点时间稳定
await asyncio.sleep(2)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)

185
experiments/dsv4_h200_vllm_tp8_custom_bench/config.env
View File

@ -32,51 +32,148 @@ LB_STRATEGY="round_robin"
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
# For single-service TP=8 we cover standard concurrency gradients.
# concurrency input_len output_len num_prompts
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 1024 128 20"
"1 1024 256 20"
"1 1024 512 20"
"1 1024 1024 20"
"1 1024 2048 20"
"1 1024 4096 20"
"8 1024 128 160"
"8 1024 256 160"
"8 1024 512 160"
"8 1024 1024 160"
"8 1024 2048 160"
"8 1024 4096 160"
"32 1024 128 640"
"32 1024 256 640"
"32 1024 512 640"
"32 1024 1024 640"
"32 1024 2048 640"
"32 1024 4096 640"
"128 1024 128 2560"
"128 1024 256 2560"
"128 1024 512 2560"
"128 1024 1024 2560"
"128 1024 2048 2560"
"128 1024 4096 2560"
"1 2048 128 20"
"1 2048 256 20"
"1 2048 512 20"
"1 2048 1024 20"
"1 2048 2048 20"
"1 2048 4096 20"
"8 2048 128 160"
"8 2048 256 160"
"8 2048 512 160"
"8 2048 1024 160"
"8 2048 2048 160"
"8 2048 4096 160"
"32 2048 128 640"
"32 2048 256 640"
"32 2048 512 640"
"32 2048 1024 640"
"32 2048 2048 640"
"32 2048 4096 640"
"128 2048 128 2560"
"128 2048 256 2560"
"128 2048 512 2560"
"128 2048 1024 2560"
"128 2048 2048 2560"
"128 2048 4096 2560"
"1 4096 128 20"
"1 4096 256 20"
"1 4096 512 20"
"1 4096 1024 20"
"1 4096 2048 20"
"1 4096 4096 20"
"8 4096 128 160"
"8 4096 256 160"
"8 4096 512 160"
"8 4096 1024 160"
"8 4096 2048 160"
"8 4096 4096 160"
"32 4096 128 640"
"32 4096 256 640"
"32 4096 512 640"
"32 4096 1024 640"
"32 4096 2048 640"
"32 4096 4096 640"
"128 4096 128 2560"
"128 4096 256 2560"
"128 4096 512 2560"
"128 4096 1024 2560"
"128 4096 2048 2560"
"128 4096 4096 2560"
"1 16384 128 20"
"1 16384 256 20"
"1 16384 512 20"
"1 16384 1024 20"
"1 16384 2048 20"
"8 16384 128 160"
"8 16384 256 160"
"8 16384 512 160"
"8 16384 1024 160"
"8 16384 2048 160"
"32 16384 128 640"
"32 16384 256 640"
"32 16384 512 640"
"32 16384 1024 640"
"32 16384 2048 640"
"128 16384 128 2560"
"128 16384 256 2560"
"128 16384 512 2560"
"128 16384 1024 2560"
"128 16384 2048 2560"
"1 65536 128 20"
"1 65536 256 20"
"1 65536 512 20"
"1 65536 1024 20"
"8 65536 128 160"
"8 65536 256 160"
"8 65536 512 160"
"8 65536 1024 160"
"32 65536 1024 160"
"1 131072 128 20"
"1 131072 256 20"
"1 131072 512 20"
"8 131072 512 160"
"1 262144 256 20"
"8 262144 128 160"
"1 524288 128 20"
)
# Server start script bundled with this experiment.

129
experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md Normal file
View File

@ -0,0 +1,129 @@
# H200 vLLM TP=2 Custom Benchmark Report
- **Client**: `bench_client.py` (async OpenAI API, per-request timing)
- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)
- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756`
## Results
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
| c128_i1024_o1024 | 128 | 1024 | 1024 | 206.75 | 2560 | 0 | 12.38 | 12679.34 | 2588.49 | 15267.83 | 355.51 | 541.00 | 3399.25 | 46.35 | 50.58 | 52.54 | 9995.61 | 13713.98 | 15284.95 | ✅ |
| c128_i1024_o128 | 128 | 1024 | 128 | 133.01 | 2560 | 0 | 19.25 | 19708.67 | 2459.70 | 22168.36 | 1183.32 | 1479.29 | 3426.82 | 41.60 | 47.50 | 49.22 | 6458.71 | 7012.22 | 8759.52 | ✅ |
| c128_i1024_o2048 | 128 | 1024 | 2048 | 208.80 | 2560 | 0 | 12.26 | 12554.51 | 2547.16 | 15101.67 | 351.46 | 493.93 | 3429.21 | 47.17 | 52.36 | 59.29 | 10106.58 | 13980.86 | 15725.90 | ✅ |
| c128_i1024_o256 | 128 | 1024 | 256 | 205.11 | 2560 | 0 | 12.48 | 12780.63 | 2561.23 | 15341.86 | 351.22 | 456.94 | 3426.54 | 46.93 | 50.36 | 51.39 | 9935.31 | 12606.55 | 13028.92 | ✅ |
| c128_i1024_o4096 | 128 | 1024 | 4096 | 208.63 | 2560 | 0 | 12.27 | 12564.96 | 2562.18 | 15127.14 | 352.13 | 475.05 | 3401.06 | 46.82 | 50.35 | 51.59 | 10079.33 | 13938.76 | 15626.71 | ✅ |
| c128_i1024_o512 | 128 | 1024 | 512 | 210.21 | 2560 | 0 | 12.18 | 12470.38 | 2534.38 | 15004.76 | 355.66 | 519.46 | 3397.33 | 47.25 | 52.84 | 55.05 | 10138.54 | 13931.68 | 15398.92 | ✅ |
| c128_i16384_o128 | 128 | 16384 | 128 | 1359.07 | 2560 | 0 | 1.88 | 30861.66 | 240.91 | 31102.57 | 34308.65 | 34726.75 | 53850.14 | 257.93 | 261.13 | 261.25 | 67037.51 | 67882.99 | 86482.77 | ❌ |
| c128_i16384_o256 | 128 | 16384 | 256 | 1383.21 | 2560 | 0 | 1.85 | 30323.03 | 393.12 | 30716.14 | 13674.31 | 16619.00 | 53959.97 | 259.70 | 265.74 | 266.01 | 68522.28 | 82348.90 | 113143.30 | ❌ |
| c128_i16384_o512 | 128 | 16384 | 512 | 1394.58 | 2560 | 0 | 1.84 | 30075.73 | 449.14 | 30524.87 | 6929.19 | 14006.30 | 53929.18 | 256.42 | 266.39 | 266.68 | 68997.86 | 134484.18 | 142208.84 | ❌ |
| c128_i2048_o1024 | 128 | 2048 | 1024 | 274.18 | 2560 | 0 | 9.34 | 19121.84 | 2068.13 | 21189.97 | 541.55 | 764.67 | 6604.67 | 57.69 | 62.18 | 65.53 | 13253.98 | 21801.42 | 28048.56 | ⚠️ |
| c128_i2048_o128 | 128 | 2048 | 128 | 212.99 | 2560 | 0 | 12.02 | 24615.40 | 1530.02 | 26145.42 | 1293.09 | 1997.49 | 6703.31 | 72.33 | 80.38 | 81.78 | 10431.32 | 11370.95 | 15884.14 | ⚠️ |
| c128_i2048_o2048 | 128 | 2048 | 2048 | 272.06 | 2560 | 0 | 9.41 | 19270.69 | 2055.16 | 21325.85 | 542.66 | 696.46 | 6595.59 | 58.02 | 62.67 | 65.23 | 13143.51 | 21512.40 | 28583.36 | ⚠️ |
| c128_i2048_o256 | 128 | 2048 | 256 | 263.83 | 2560 | 0 | 9.70 | 19872.28 | 1986.56 | 21858.84 | 552.74 | 766.94 | 6667.42 | 60.36 | 64.80 | 67.61 | 12841.30 | 16455.00 | 19770.97 | ⚠️ |
| c128_i2048_o4096 | 128 | 2048 | 4096 | 274.18 | 2560 | 0 | 9.34 | 19121.82 | 2075.59 | 21197.41 | 538.85 | 772.70 | 6666.24 | 57.59 | 61.73 | 65.98 | 13271.96 | 22132.76 | 28679.85 | ⚠️ |
| c128_i2048_o512 | 128 | 2048 | 512 | 271.80 | 2560 | 0 | 9.42 | 19289.73 | 2074.26 | 21363.99 | 544.24 | 780.74 | 6725.67 | 57.75 | 61.62 | 65.40 | 13185.12 | 21822.20 | 28335.44 | ⚠️ |
| c128_i4096_o1024 | 128 | 4096 | 1024 | 424.83 | 2560 | 0 | 6.03 | 24682.53 | 1273.81 | 25956.34 | 1014.26 | 1488.91 | 13325.90 | 93.93 | 103.74 | 115.24 | 20697.67 | 35507.95 | 47170.14 | ⚠️ |
| c128_i4096_o128 | 128 | 4096 | 128 | 371.42 | 2560 | 0 | 6.89 | 28231.71 | 873.71 | 29105.42 | 1687.74 | 2056.57 | 13344.42 | 132.08 | 142.11 | 145.58 | 18310.58 | 19462.37 | 30169.28 | ⚠️ |
| c128_i4096_o2048 | 128 | 4096 | 2048 | 425.25 | 2560 | 0 | 6.02 | 24657.77 | 1288.03 | 25945.80 | 1019.46 | 1464.89 | 13332.86 | 93.13 | 101.52 | 111.69 | 20779.35 | 36358.76 | 50447.50 | ⚠️ |
| c128_i4096_o256 | 128 | 4096 | 256 | 413.80 | 2560 | 0 | 6.19 | 25340.14 | 1217.79 | 26557.93 | 1065.75 | 1554.31 | 13333.89 | 98.37 | 108.53 | 114.97 | 20311.65 | 26976.60 | 33054.57 | ⚠️ |
| c128_i4096_o4096 | 128 | 4096 | 4096 | 423.76 | 2560 | 0 | 6.04 | 24744.50 | 1282.56 | 26027.06 | 1040.84 | 1941.31 | 13328.94 | 93.34 | 101.81 | 111.13 | 20723.12 | 35255.54 | 48374.31 | ⚠️ |
| c128_i4096_o512 | 128 | 4096 | 512 | 423.70 | 2560 | 0 | 6.04 | 24748.17 | 1274.42 | 26022.58 | 1032.52 | 1562.56 | 13323.34 | 93.86 | 103.68 | 112.51 | 20669.59 | 35396.55 | 49034.77 | ⚠️ |
| c128_i512_o256 | 128 | 512 | 256 | 198.05 | 2560 | 0 | 12.93 | 6618.16 | 2673.83 | 9291.99 | 290.66 | 501.00 | 1944.87 | 45.16 | 52.68 | 57.19 | 9585.92 | 13135.34 | 14237.26 | ✅ |
| c16_i512_o256 | 16 | 512 | 256 | 70.00 | 320 | 0 | 4.57 | 2340.54 | 954.25 | 3294.79 | 155.99 | 299.17 | 421.01 | 14.77 | 16.43 | 17.44 | 3227.91 | 4110.18 | 4253.24 | ✅ |
| c1_i1024_o1024 | 1 | 1024 | 1024 | 41.00 | 20 | 0 | 0.49 | 499.49 | 102.02 | 601.51 | 113.27 | 116.29 | 137.79 | 6.73 | 6.74 | 6.74 | 1513.65 | 1949.27 | 1998.66 | ✅ |
| c1_i1024_o128 | 1 | 1024 | 128 | 26.62 | 20 | 0 | 0.75 | 769.29 | 96.16 | 865.45 | 112.10 | 114.09 | 127.55 | 6.69 | 6.70 | 6.70 | 961.87 | 964.11 | 977.09 | ✅ |
| c1_i1024_o2048 | 1 | 1024 | 2048 | 40.37 | 20 | 0 | 0.50 | 507.26 | 100.39 | 607.65 | 113.54 | 118.17 | 129.38 | 6.73 | 6.75 | 6.75 | 1470.10 | 1949.87 | 2093.40 | ✅ |
| c1_i1024_o256 | 1 | 1024 | 256 | 39.66 | 20 | 0 | 0.50 | 516.37 | 101.41 | 617.77 | 112.03 | 113.19 | 128.94 | 6.73 | 6.74 | 6.74 | 1457.72 | 1819.60 | 1819.96 | ✅ |
| c1_i1024_o4096 | 1 | 1024 | 4096 | 39.07 | 20 | 0 | 0.51 | 524.19 | 100.38 | 624.57 | 126.46 | 178.17 | 178.35 | 6.73 | 6.74 | 6.75 | 1439.54 | 1773.78 | 1831.83 | ✅ |
| c1_i1024_o512 | 1 | 1024 | 512 | 38.86 | 20 | 0 | 0.51 | 527.06 | 98.62 | 625.68 | 113.09 | 114.42 | 136.00 | 6.73 | 6.74 | 6.75 | 1396.05 | 1797.45 | 1820.39 | ✅ |
| c1_i16384_o1024 | 1 | 16384 | 1024 | 60.02 | 20 | 0 | 0.33 | 5459.87 | 74.28 | 5534.15 | 621.87 | 633.10 | 637.21 | 6.82 | 6.83 | 6.85 | 2136.32 | 2962.66 | 3211.92 | ✅ |
| c1_i16384_o128 | 1 | 16384 | 128 | 41.67 | 20 | 0 | 0.48 | 7862.85 | 60.88 | 7923.72 | 628.49 | 631.58 | 636.36 | 6.77 | 6.81 | 6.81 | 1480.09 | 1482.52 | 1482.98 | ✅ |
| c1_i16384_o2048 | 1 | 16384 | 2048 | 63.80 | 20 | 0 | 0.31 | 5136.27 | 80.76 | 5217.03 | 630.15 | 636.36 | 636.98 | 6.82 | 6.83 | 6.85 | 2381.45 | 4577.47 | 5209.66 | ✅ |
| c1_i16384_o256 | 1 | 16384 | 256 | 56.44 | 20 | 0 | 0.35 | 5805.62 | 73.65 | 5879.27 | 630.48 | 638.61 | 639.04 | 6.82 | 6.85 | 6.85 | 2040.24 | 2358.86 | 2362.47 | ✅ |
| c1_i16384_o512 | 1 | 16384 | 512 | 70.93 | 20 | 0 | 0.28 | 4619.51 | 85.87 | 4705.38 | 621.45 | 629.96 | 645.42 | 6.82 | 6.85 | 6.85 | 2692.18 | 4094.39 | 4094.98 | ✅ |
| c1_i2048_o1024 | 1 | 2048 | 1024 | 43.34 | 20 | 0 | 0.46 | 945.05 | 102.00 | 1047.06 | 123.15 | 127.95 | 148.69 | 6.80 | 6.81 | 6.81 | 1619.33 | 2410.40 | 2903.39 | ✅ |
| c1_i2048_o128 | 1 | 2048 | 128 | 27.02 | 20 | 0 | 0.74 | 1515.96 | 94.12 | 1610.08 | 122.85 | 126.65 | 147.47 | 6.77 | 6.77 | 6.82 | 976.42 | 983.68 | 1006.22 | ✅ |
| c1_i2048_o2048 | 1 | 2048 | 2048 | 43.35 | 20 | 0 | 0.46 | 944.86 | 101.45 | 1046.31 | 123.06 | 128.19 | 147.54 | 6.80 | 6.83 | 6.84 | 1611.88 | 2108.88 | 2496.01 | ✅ |
| c1_i2048_o256 | 1 | 2048 | 256 | 37.22 | 20 | 0 | 0.54 | 1100.59 | 96.68 | 1197.27 | 122.80 | 125.69 | 145.02 | 6.81 | 6.82 | 6.83 | 1340.69 | 1621.05 | 1775.55 | ✅ |
| c1_i2048_o4096 | 1 | 2048 | 4096 | 40.11 | 20 | 0 | 0.50 | 1021.08 | 101.61 | 1122.69 | 123.10 | 127.51 | 146.72 | 6.80 | 6.82 | 6.82 | 1502.66 | 2076.61 | 2137.65 | ✅ |
| c1_i2048_o512 | 1 | 2048 | 512 | 37.02 | 20 | 0 | 0.54 | 1106.44 | 96.52 | 1202.96 | 122.80 | 127.33 | 146.10 | 6.81 | 6.82 | 6.82 | 1331.89 | 1558.94 | 1578.90 | ✅ |
| c1_i4096_o1024 | 1 | 4096 | 1024 | 43.12 | 20 | 0 | 0.46 | 1899.75 | 99.83 | 1999.59 | 186.35 | 190.63 | 190.95 | 6.80 | 6.81 | 6.81 | 1644.18 | 2109.89 | 2311.93 | ✅ |
| c1_i4096_o128 | 1 | 4096 | 128 | 28.91 | 20 | 0 | 0.69 | 2833.86 | 88.56 | 2922.42 | 185.86 | 190.38 | 191.25 | 6.75 | 6.76 | 6.76 | 1043.43 | 1045.36 | 1047.58 | ✅ |
| c1_i4096_o2048 | 1 | 4096 | 2048 | 42.94 | 20 | 0 | 0.47 | 1907.88 | 96.51 | 2004.39 | 186.77 | 193.30 | 193.74 | 6.80 | 6.81 | 6.81 | 1589.25 | 2097.82 | 2452.99 | ✅ |
| c1_i4096_o256 | 1 | 4096 | 256 | 40.43 | 20 | 0 | 0.49 | 2026.29 | 93.65 | 2119.94 | 189.05 | 207.59 | 207.83 | 6.80 | 6.81 | 6.82 | 1469.51 | 1913.76 | 1913.87 | ✅ |
| c1_i4096_o4096 | 1 | 4096 | 4096 | 43.12 | 20 | 0 | 0.46 | 1899.82 | 96.29 | 1996.11 | 187.93 | 205.29 | 207.29 | 6.80 | 6.82 | 6.82 | 1593.63 | 2383.61 | 2729.94 | ✅ |
| c1_i4096_o512 | 1 | 4096 | 512 | 41.37 | 20 | 0 | 0.48 | 1980.16 | 89.19 | 2069.35 | 186.00 | 187.35 | 190.66 | 6.81 | 6.81 | 6.82 | 1434.80 | 2102.19 | 2396.89 | ✅ |
| c1_i512_o256 | 1 | 512 | 256 | 39.88 | 20 | 0 | 0.50 | 256.78 | 96.80 | 353.58 | 118.18 | 176.60 | 184.48 | 6.72 | 6.75 | 6.75 | 1408.44 | 1822.85 | 1871.30 | ✅ |
| c2_i512_o256 | 2 | 512 | 256 | 41.58 | 40 | 0 | 0.96 | 492.58 | 196.05 | 688.62 | 173.55 | 211.08 | 357.86 | 7.61 | 8.16 | 8.40 | 1706.70 | 2108.24 | 2165.31 | ✅ |
| c32_i1024_o1024 | 32 | 1024 | 1024 | 101.71 | 640 | 0 | 6.29 | 6443.13 | 1317.59 | 7760.73 | 197.93 | 311.27 | 984.94 | 22.03 | 23.78 | 24.21 | 4796.09 | 6445.23 | 7146.16 | ✅ |
| c32_i1024_o128 | 32 | 1024 | 128 | 53.75 | 640 | 0 | 11.91 | 12192.47 | 1523.87 | 13716.35 | 681.10 | 1046.63 | 1091.53 | 14.34 | 16.98 | 18.60 | 2502.18 | 2579.44 | 2641.32 | ✅ |
| c32_i1024_o2048 | 32 | 1024 | 2048 | 101.76 | 640 | 0 | 6.29 | 6440.44 | 1316.74 | 7757.18 | 204.19 | 335.85 | 1145.74 | 21.89 | 24.02 | 24.53 | 4767.74 | 6630.19 | 7109.51 | ✅ |
| c32_i1024_o256 | 32 | 1024 | 256 | 99.60 | 640 | 0 | 6.43 | 6579.98 | 1309.14 | 7889.12 | 200.93 | 331.50 | 921.88 | 22.15 | 24.50 | 25.32 | 4694.16 | 6056.40 | 6274.46 | ✅ |
| c32_i1024_o4096 | 32 | 1024 | 4096 | 103.10 | 640 | 0 | 6.21 | 6356.26 | 1309.63 | 7665.88 | 205.97 | 316.61 | 1157.52 | 22.04 | 24.05 | 24.79 | 4833.69 | 6458.13 | 7314.04 | ✅ |
| c32_i1024_o512 | 32 | 1024 | 512 | 103.06 | 640 | 0 | 6.21 | 6359.18 | 1304.32 | 7663.50 | 196.61 | 340.27 | 945.16 | 22.38 | 25.38 | 27.97 | 4872.21 | 6788.10 | 7689.83 | ✅ |
| c32_i16384_o1024 | 32 | 16384 | 1024 | 398.20 | 640 | 0 | 1.61 | 26333.15 | 402.85 | 26735.99 | 1772.26 | 3387.53 | 13518.46 | 70.46 | 85.49 | 101.84 | 19184.30 | 36508.49 | 54198.78 | ❌ |
| c32_i16384_o128 | 32 | 16384 | 128 | 359.20 | 640 | 0 | 1.78 | 29191.93 | 227.55 | 29419.48 | 3438.60 | 3983.92 | 13516.26 | 111.13 | 127.95 | 130.66 | 17518.50 | 19731.45 | 27923.40 | ❌ |
| c32_i16384_o2048 | 32 | 16384 | 2048 | 398.63 | 640 | 0 | 1.61 | 26304.81 | 403.52 | 26708.32 | 1715.04 | 2963.25 | 13514.42 | 70.47 | 86.65 | 95.98 | 19175.98 | 37536.35 | 50504.81 | ⚠️ |
| c32_i16384_o256 | 32 | 16384 | 256 | 383.92 | 640 | 0 | 1.67 | 27312.05 | 348.16 | 27660.21 | 1826.81 | 3388.69 | 13517.47 | 80.79 | 92.93 | 101.63 | 18646.62 | 24405.54 | 29695.09 | ❌ |
| c32_i16384_o512 | 32 | 16384 | 512 | 393.37 | 640 | 0 | 1.63 | 26656.12 | 383.78 | 27039.90 | 1747.04 | 3400.82 | 13513.48 | 73.78 | 90.30 | 96.96 | 19000.89 | 36338.55 | 41160.95 | ❌ |
| c32_i2048_o1024 | 32 | 2048 | 1024 | 112.12 | 640 | 0 | 5.71 | 11690.22 | 1270.45 | 12960.67 | 246.53 | 373.10 | 1896.26 | 22.21 | 24.25 | 25.25 | 5165.11 | 8407.83 | 11262.12 | ✅ |
| c32_i2048_o128 | 32 | 2048 | 128 | 73.91 | 640 | 0 | 8.66 | 17733.36 | 1105.41 | 18838.77 | 904.68 | 1464.61 | 1844.76 | 20.43 | 26.38 | 27.06 | 3492.80 | 3746.52 | 4348.11 | ✅ |
| c32_i2048_o2048 | 32 | 2048 | 2048 | 111.79 | 640 | 0 | 5.73 | 11725.08 | 1282.42 | 13007.50 | 246.50 | 442.18 | 1847.04 | 22.17 | 24.28 | 25.62 | 5185.73 | 8810.37 | 10858.07 | ✅ |
| c32_i2048_o256 | 32 | 2048 | 256 | 103.61 | 640 | 0 | 6.18 | 12650.11 | 1258.11 | 13908.22 | 240.98 | 367.88 | 1848.70 | 22.80 | 24.74 | 25.47 | 4859.84 | 6298.45 | 6687.99 | ✅ |
| c32_i2048_o4096 | 32 | 2048 | 4096 | 106.84 | 640 | 0 | 5.99 | 12268.59 | 1282.15 | 13550.74 | 250.26 | 439.47 | 1837.79 | 22.45 | 24.65 | 25.84 | 5036.07 | 8060.13 | 10159.85 | ✅ |
| c32_i2048_o512 | 32 | 2048 | 512 | 111.06 | 640 | 0 | 5.76 | 11802.41 | 1281.65 | 13084.06 | 242.26 | 437.24 | 1893.17 | 22.15 | 24.51 | 25.55 | 5136.64 | 8816.12 | 11310.11 | ✅ |
| c32_i4096_o1024 | 32 | 4096 | 1024 | 141.80 | 640 | 0 | 4.51 | 18486.39 | 947.32 | 19433.71 | 432.85 | 813.49 | 3483.06 | 30.04 | 33.92 | 36.32 | 6702.72 | 11605.60 | 14590.75 | ✅ |
| c32_i4096_o128 | 32 | 4096 | 128 | 114.34 | 640 | 0 | 5.60 | 22925.90 | 710.98 | 23636.88 | 1053.81 | 1909.50 | 3485.20 | 35.10 | 41.98 | 43.26 | 5478.89 | 6106.50 | 7458.03 | ✅ |
| c32_i4096_o2048 | 32 | 4096 | 2048 | 144.45 | 640 | 0 | 4.43 | 18147.38 | 939.67 | 19087.05 | 434.39 | 747.31 | 3483.20 | 29.88 | 33.53 | 35.93 | 6700.04 | 10714.15 | 14733.26 | ✅ |
| c32_i4096_o256 | 32 | 4096 | 256 | 137.28 | 640 | 0 | 4.66 | 19095.92 | 921.51 | 20017.42 | 441.00 | 731.75 | 3478.68 | 30.98 | 34.42 | 36.58 | 6528.50 | 8638.60 | 10034.50 | ✅ |
| c32_i4096_o4096 | 32 | 4096 | 4096 | 143.01 | 640 | 0 | 4.48 | 18330.54 | 961.50 | 19292.04 | 429.61 | 749.16 | 3469.34 | 29.60 | 33.17 | 35.01 | 6743.49 | 10500.86 | 14281.37 | ✅ |
| c32_i4096_o512 | 32 | 4096 | 512 | 143.19 | 640 | 0 | 4.47 | 18307.18 | 929.29 | 19236.47 | 430.74 | 691.68 | 3532.95 | 30.57 | 34.15 | 36.96 | 6735.35 | 10401.62 | 14892.06 | ✅ |
| c32_i512_o256 | 32 | 512 | 256 | 98.73 | 640 | 0 | 6.48 | 3319.01 | 1330.82 | 4649.83 | 186.98 | 378.34 | 678.28 | 21.82 | 23.82 | 24.54 | 4649.84 | 6039.51 | 6163.96 | ✅ |
| c4_i512_o256 | 4 | 512 | 256 | 46.00 | 80 | 0 | 1.74 | 890.42 | 360.47 | 1250.89 | 164.64 | 347.47 | 374.55 | 8.92 | 10.51 | 11.09 | 1997.44 | 2523.64 | 2675.43 | ✅ |
| c64_i512_o256 | 64 | 512 | 256 | 140.30 | 1280 | 0 | 9.12 | 4671.07 | 1901.53 | 6572.60 | 229.06 | 453.78 | 1147.81 | 31.26 | 35.35 | 36.92 | 6712.71 | 8741.37 | 9237.35 | ✅ |
| c8_i1024_o1024 | 8 | 1024 | 1024 | 54.01 | 160 | 0 | 2.96 | 3033.30 | 621.77 | 3655.07 | 144.65 | 218.00 | 394.00 | 10.80 | 11.82 | 12.24 | 2395.48 | 3214.61 | 3506.80 | ✅ |
| c8_i1024_o128 | 8 | 1024 | 128 | 30.89 | 160 | 0 | 5.18 | 5304.66 | 662.82 | 5967.48 | 307.64 | 345.56 | 389.57 | 8.30 | 9.77 | 9.80 | 1360.75 | 1376.96 | 1412.43 | ✅ |
| c8_i1024_o2048 | 8 | 1024 | 2048 | 53.44 | 160 | 0 | 2.99 | 3065.61 | 627.08 | 3692.69 | 147.14 | 284.31 | 375.40 | 10.76 | 11.60 | 12.31 | 2391.87 | 3190.58 | 3558.55 | ✅ |
| c8_i1024_o256 | 8 | 1024 | 256 | 52.32 | 160 | 0 | 3.06 | 3131.63 | 620.55 | 3752.18 | 144.06 | 216.85 | 383.28 | 10.89 | 11.75 | 12.29 | 2345.43 | 2965.11 | 3042.16 | ✅ |
| c8_i1024_o4096 | 8 | 1024 | 4096 | 55.32 | 160 | 0 | 2.89 | 2961.58 | 621.49 | 3583.07 | 144.43 | 220.03 | 380.43 | 10.73 | 11.56 | 12.09 | 2438.42 | 3382.06 | 3756.38 | ✅ |
| c8_i1024_o512 | 8 | 1024 | 512 | 54.03 | 160 | 0 | 2.96 | 3032.14 | 632.21 | 3664.35 | 144.40 | 216.10 | 375.86 | 10.74 | 11.48 | 11.76 | 2425.82 | 3208.35 | 3616.77 | ✅ |
| c8_i16384_o1024 | 8 | 16384 | 1024 | 138.97 | 160 | 0 | 1.15 | 18863.47 | 287.59 | 19151.06 | 967.72 | 1600.40 | 3663.33 | 21.47 | 27.32 | 29.59 | 6298.76 | 12516.84 | 14588.60 | ✅ |
| c8_i16384_o128 | 8 | 16384 | 128 | 111.93 | 160 | 0 | 1.43 | 23420.44 | 181.33 | 23601.76 | 2556.64 | 4119.31 | 4136.10 | 20.74 | 33.97 | 34.00 | 5167.54 | 5417.75 | 5933.67 | ⚠️ |
| c8_i16384_o2048 | 8 | 16384 | 2048 | 136.26 | 160 | 0 | 1.17 | 19238.95 | 275.01 | 19513.96 | 961.94 | 1572.87 | 3673.56 | 22.52 | 28.43 | 30.68 | 6163.97 | 11684.40 | 14272.99 | ✅ |
| c8_i16384_o256 | 8 | 16384 | 256 | 127.82 | 160 | 0 | 1.25 | 20509.06 | 253.69 | 20762.74 | 1027.09 | 1938.52 | 3668.72 | 24.19 | 29.35 | 30.47 | 5890.64 | 7584.77 | 7970.70 | ✅ |
| c8_i16384_o512 | 8 | 16384 | 512 | 136.61 | 160 | 0 | 1.17 | 19188.83 | 286.09 | 19474.92 | 969.45 | 1904.35 | 3667.98 | 21.65 | 26.89 | 29.25 | 6250.80 | 10966.73 | 12572.72 | ✅ |
| c8_i2048_o1024 | 8 | 2048 | 1024 | 59.22 | 160 | 0 | 2.70 | 5533.53 | 606.82 | 6140.34 | 155.89 | 244.40 | 621.82 | 10.83 | 11.86 | 12.11 | 2572.18 | 4486.64 | 5823.30 | ✅ |
| c8_i2048_o128 | 8 | 2048 | 128 | 36.79 | 160 | 0 | 4.35 | 8907.76 | 554.37 | 9462.13 | 337.85 | 507.02 | 613.47 | 10.27 | 12.09 | 12.51 | 1636.65 | 1709.00 | 1726.45 | ✅ |
| c8_i2048_o2048 | 8 | 2048 | 2048 | 57.55 | 160 | 0 | 2.78 | 5693.86 | 626.97 | 6320.83 | 156.06 | 275.62 | 603.04 | 10.76 | 11.60 | 11.99 | 2571.87 | 4101.50 | 5201.48 | ✅ |
| c8_i2048_o256 | 8 | 2048 | 256 | 52.19 | 160 | 0 | 3.07 | 6279.15 | 604.00 | 6883.15 | 158.73 | 268.03 | 611.54 | 11.13 | 12.15 | 12.45 | 2334.26 | 3004.26 | 3124.59 | ✅ |
| c8_i2048_o4096 | 8 | 2048 | 4096 | 59.22 | 160 | 0 | 2.70 | 5533.71 | 614.83 | 6148.54 | 164.46 | 304.53 | 685.46 | 10.84 | 11.77 | 11.98 | 2624.68 | 4094.22 | 5661.19 | ✅ |
| c8_i2048_o512 | 8 | 2048 | 512 | 57.76 | 160 | 0 | 2.77 | 5672.85 | 605.80 | 6278.66 | 154.18 | 248.11 | 617.88 | 10.86 | 11.81 | 12.00 | 2521.32 | 3808.31 | 4671.03 | ✅ |
| c8_i4096_o1024 | 8 | 4096 | 1024 | 64.92 | 160 | 0 | 2.46 | 10094.27 | 518.13 | 10612.40 | 256.33 | 442.48 | 1016.15 | 12.36 | 13.77 | 14.46 | 2846.88 | 4602.44 | 5170.71 | ✅ |
| c8_i4096_o128 | 8 | 4096 | 128 | 46.47 | 160 | 0 | 3.44 | 14102.60 | 440.04 | 14542.64 | 707.03 | 1047.28 | 1071.50 | 10.92 | 15.03 | 15.06 | 2091.88 | 2145.56 | 2225.52 | ✅ |
| c8_i4096_o2048 | 8 | 4096 | 2048 | 66.29 | 160 | 0 | 2.41 | 9886.22 | 513.33 | 10399.56 | 253.25 | 423.32 | 1022.59 | 12.39 | 14.05 | 14.66 | 2865.36 | 4853.71 | 6028.24 | ✅ |
| c8_i4096_o256 | 8 | 4096 | 256 | 62.26 | 160 | 0 | 2.57 | 10526.27 | 516.50 | 11042.77 | 256.86 | 457.13 | 1013.55 | 12.63 | 14.05 | 14.57 | 2789.27 | 3625.36 | 3833.62 | ✅ |
| c8_i4096_o4096 | 8 | 4096 | 4096 | 64.28 | 160 | 0 | 2.49 | 10195.14 | 543.84 | 10738.99 | 248.19 | 462.57 | 1014.75 | 12.27 | 13.68 | 14.32 | 2921.13 | 5570.16 | 8112.52 | ✅ |
| c8_i4096_o512 | 8 | 4096 | 512 | 65.25 | 160 | 0 | 2.45 | 10043.48 | 538.74 | 10582.22 | 247.69 | 442.69 | 1015.54 | 12.25 | 13.71 | 14.06 | 2931.06 | 5124.89 | 6513.32 | ✅ |
| c8_i512_o256 | 8 | 512 | 256 | 52.73 | 160 | 0 | 3.03 | 1553.68 | 629.48 | 2183.16 | 135.86 | 271.88 | 289.07 | 10.80 | 11.89 | 12.07 | 2369.82 | 3039.64 | 3126.47 | ✅ |
| c128_i16384_o1024 | 128 | 16384 | 1024 | 1401.24 | 2560 | 0 | 1.83 | 29932.73 | 453.17 | 30385.90 | 6827.03 | 13719.55 | 53845.73 | 254.25 | 266.59 | 267.01 | 69085.30 | 135447.23 | 185844.63 | ❌ |
| c128_i16384_o2048 | 128 | 16384 | 2048 | 1400.62 | 2560 | 0 | 1.83 | 29945.97 | 468.45 | 30414.41 | 5874.33 | 13475.14 | 53934.11 | 249.38 | 266.26 | 266.71 | 69277.40 | 141505.20 | 189264.63 | ❌ |
| c1_i131072_o128 | 1 | 131072 | 128 | 182.19 | 20 | 0 | 0.11 | 14388.29 | 12.72 | 14401.01 | 5392.41 | 5419.10 | 5420.54 | 6.96 | 6.97 | 6.98 | 6192.49 | 6221.42 | 6221.43 | ⚠️ |
| c1_i131072_o256 | 1 | 131072 | 256 | 195.04 | 20 | 0 | 0.10 | 13440.21 | 19.01 | 13459.22 | 5397.54 | 5431.66 | 5440.22 | 7.00 | 7.03 | 7.03 | 6686.72 | 7103.29 | 7119.59 | ⚠️ |
| c1_i131072_o512 | 1 | 131072 | 512 | 201.48 | 20 | 0 | 0.10 | 13010.69 | 20.19 | 13030.87 | 5500.61 | 5555.72 | 5562.25 | 7.00 | 7.02 | 7.03 | 6917.34 | 7364.34 | 8109.96 | ⚠️ |
| c1_i262144_o256 | 1 | 262144 | 256 | 426.96 | 20 | 0 | 0.05 | 12279.65 | 8.19 | 12287.84 | 13218.70 | 13504.54 | 13586.50 | 7.22 | 7.32 | 7.36 | 14470.75 | 14834.79 | 14835.15 | ⚠️ |
| c1_i524288_o128 | 1 | 524288 | 128 | 1098.28 | 20 | 0 | 0.02 | 9547.46 | 1.62 | 9549.08 | 36239.34 | 36595.44 | 37102.25 | 9.37 | 9.80 | 37.00 | 36907.34 | 37308.40 | 37807.76 | ⚠️ |
| c1_i65536_o1024 | 1 | 65536 | 1024 | 114.83 | 20 | 0 | 0.17 | 11414.14 | 39.37 | 11453.51 | 2462.78 | 2475.26 | 2495.15 | 6.92 | 6.93 | 6.93 | 4020.21 | 5428.03 | 7058.64 | ✅ |
| c1_i65536_o128 | 1 | 65536 | 128 | 96.37 | 20 | 0 | 0.21 | 13600.71 | 24.89 | 13625.61 | 2497.10 | 2528.43 | 2558.93 | 6.87 | 6.87 | 6.88 | 3313.69 | 3344.68 | 3373.65 | ✅ |
| c1_i65536_o256 | 1 | 65536 | 256 | 105.96 | 20 | 0 | 0.19 | 12370.49 | 32.66 | 12403.15 | 2460.46 | 2472.33 | 2483.17 | 6.92 | 6.93 | 6.93 | 3650.21 | 4159.14 | 4159.75 | ✅ |
| c1_i65536_o512 | 1 | 65536 | 512 | 108.81 | 20 | 0 | 0.18 | 12045.94 | 34.57 | 12080.52 | 2461.82 | 2468.77 | 2498.35 | 6.92 | 6.93 | 6.93 | 3756.97 | 4315.50 | 4593.51 | ✅ |
| c32_i65536_o1024 | 32 | 65536 | 1024 | 395.45 | 160 | 0 | 0.40 | 26515.91 | 89.22 | 26605.13 | 18530.86 | 54451.46 | 68783.28 | 252.00 | 283.71 | 284.41 | 72731.85 | 117936.92 | 151338.20 | ❌ |
| c8_i131072_o512 | 8 | 131072 | 512 | 887.91 | 160 | 0 | 0.18 | 23619.01 | 33.13 | 23652.14 | 12820.61 | 25478.95 | 32945.62 | 156.62 | 223.19 | 294.51 | 41479.53 | 66809.84 | 79279.57 | ❌ |
| c8_i262144_o128 | 8 | 262144 | 128 | 2121.21 | 160 | 0 | 0.08 | 19773.16 | 9.48 | 19782.64 | 51391.65 | 61587.85 | 80661.75 | 382.04 | 388.68 | 389.24 | 99110.05 | 110875.08 | 129961.05 | ❌ |
| c8_i65536_o1024 | 8 | 65536 | 1024 | 422.41 | 160 | 0 | 0.38 | 24823.70 | 76.89 | 24900.59 | 4665.85 | 8696.15 | 14818.98 | 73.86 | 99.00 | 113.60 | 19699.15 | 35998.21 | 56237.93 | ❌ |
| c8_i65536_o128 | 8 | 65536 | 128 | 402.35 | 160 | 0 | 0.40 | 26061.00 | 46.48 | 26107.48 | 8659.91 | 13193.45 | 15765.71 | 87.87 | 143.78 | 166.11 | 18805.63 | 21397.73 | 22483.11 | ❌ |
| c8_i65536_o256 | 8 | 65536 | 256 | 416.75 | 160 | 0 | 0.38 | 25160.80 | 67.25 | 25228.06 | 5113.03 | 10698.95 | 14785.06 | 82.53 | 113.13 | 132.88 | 19445.90 | 27248.32 | 32918.89 | ❌ |
| c8_i65536_o512 | 8 | 65536 | 512 | 422.66 | 160 | 0 | 0.38 | 24809.03 | 76.23 | 24885.26 | 4532.43 | 9015.11 | 14789.72 | 75.30 | 108.22 | 126.16 | 19689.27 | 34072.42 | 42942.27 | ❌ |
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.

6428
experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json Normal file
View File

File diff suppressed because it is too large Load Diff

1
experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh
View File

@ -71,7 +71,6 @@ data["config"] = {
"gpu_groups": "0,1,2,3,4,5,6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [

103
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md
View File

@ -1,103 +0,0 @@
# dsv4_h200_vllm_tp8_custom_bench_no_fp8
> **Note:** This variant does NOT use `--kv-cache-dtype fp8`.
## 目的
当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU：
| 服务 | 端口 | GPU 组 |
|------|------|--------|
| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 |
TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。
## 与多服务方案的区别
| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) |
|------|---------------|---------------|---------------|
| 服务数量 | 4 | 2 | 1 |
| 每服务 GPU | 2 | 4 | 8 |
| 单服务吞吐 | 较低 | 中等 | 最高 |
| 集群总吞吐 | 相近 | 相近 | 相近 |
| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch |
## 文件说明
| 文件 | 说明 |
|------|------|
| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） |
| `config.env` | 实验配置（单服务、端口 30005） |
| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 |
| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） |
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
## 快速运行
```bash
# 完整流程（自动启动服务、压测、生成报告）
bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh
# 跳过服务管理（已有服务在运行）
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh
# 单独运行压测客户端（单服务）
python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
--output-file /tmp/test.jsonl
# 解析已有结果
python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \
experiments/dsv4_h200_vllm_tp8_custom_bench/results/<RUN_ID>
```
## 场景设计
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|------|----------|----------|--------|------|
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
| 2 | 512 | 256 | 40 | 低并发 |
| 4 | 512 | 256 | 80 | 中低并发 |
| 8 | 512 | 256 | 160 | 中等并发 |
| 16 | 512 | 256 | 320 | 中高并发 |
| 32 | 512 | 256 | 640 | 高并发 |
| 64 | 512 | 256 | 1280 | 很高并发 |
| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 |
| 1 | 2048 | 512 | 20 | 长输入基线 |
| 8 | 2048 | 512 | 160 | 长输入中并发 |
| 32 | 2048 | 512 | 640 | 长输入高并发 |
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
## 输出格式
同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。
## 单独启动/停止服务
```bash
# 启动服务
bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh
# 停止（run_bench.sh 会自动停止，但也可以手动）
port=30005
pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid"
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
rm -f "$pid_file"
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
```
## 注意事项
- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。
- 模型文件路径：`/data/models/DeepSeek-V4-Flash`
- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。
- 健康检查依赖 `/health` 端点（vLLM 默认启用）。
- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。

617
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py
View File

@ -1,617 +0,0 @@
#!/usr/bin/env python3
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
Sends concurrent requests to multiple inference servers and records per-request
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
Designed for TP=2 on 8-GPU setups where 4 independent services run on
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
round-robin or random load balancing. Unhealthy services are automatically
skipped with periodic retry.
Usage:
# Single service (backward compatible)
python3 bench_client.py \
--host 127.0.0.1 --port 30005 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--output-file results.jsonl
# Multi-service (4 services on ports 30005-30008)
python3 bench_client.py \
--host 127.0.0.1 \
--ports 30005,30006,30007,30008 \
--model deepseek-v4-flash \
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
--lb-strategy round_robin \
--output-file results.jsonl
The output JSONL contains one object per request with raw timing data.
A final summary line ("completed" field) aggregates all requests.
"""
import argparse
import asyncio
import json
import random
import sys
import time
import urllib.request
from dataclasses import asdict, dataclass, field
from typing import Any
# Try aiohttp first; fall back to urllib for stdlib-only environments.
try:
import aiohttp
HAS_AIOHTTP = True
except ImportError:
HAS_AIOHTTP = False
# ---------------------------------------------------------------------------
# Data structures
# ---------------------------------------------------------------------------
@dataclass
class RequestResult:
request_id: int
input_tokens: int = 0
output_tokens: int = 0
first_token_time_ms: float = 0.0 # TTFT
e2e_latency_ms: float = 0.0 # total wall time
inter_token_latencies: list[float] = field(default_factory=list)
success: bool = False
error: str = ""
target_port: int = 0 # which service handled this request
retry_count: int = 0 # how many times this request was retried
@property
def tpot_ms(self) -> float:
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
if self.output_tokens <= 1:
return 0.0
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
@property
def mean_itl_ms(self) -> float:
if not self.inter_token_latencies:
return 0.0
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
def to_dict(self) -> dict[str, Any]:
d = asdict(self)
d["tpot_ms"] = self.tpot_ms
d["mean_itl_ms"] = self.mean_itl_ms
return d
# ---------------------------------------------------------------------------
# Health checker
# ---------------------------------------------------------------------------
class HealthChecker:
"""Periodically check service health and maintain a healthy-port list."""
def __init__(
self,
host: str,
ports: list[int],
check_interval: float = 5.0,
max_failures: int = 2,
recovery_interval: float = 10.0,
):
self.host = host
self.all_ports = ports
self.check_interval = check_interval
self.max_failures = max_failures
self.recovery_interval = recovery_interval
# port -> consecutive failure count
self._failures: dict[int, int] = {p: 0 for p in ports}
# port -> last healthy timestamp
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
self._lock = asyncio.Lock()
self._task: asyncio.Task | None = None
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
url = f"http://{self.host}:{port}/health"
try:
if HAS_AIOHTTP and session is not None:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
return resp.status == 200
else:
req = urllib.request.Request(url, method="GET")
with urllib.request.urlopen(req, timeout=5) as resp:
return resp.status == 200
except Exception:
return False
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
checks = [self._check_one(p, session) for p in self.all_ports]
results = await asyncio.gather(*checks, return_exceptions=True)
now = time.monotonic()
async with self._lock:
for port, ok in zip(self.all_ports, results):
if isinstance(ok, Exception):
ok = False
if ok:
self._failures[port] = 0
self._last_healthy[port] = now
else:
self._failures[port] += 1
async def start(self, session: aiohttp.ClientSession | None) -> None:
async def _loop() -> None:
while True:
await asyncio.sleep(self.check_interval)
await self._check_all(session)
self._task = asyncio.create_task(_loop())
async def stop(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
@property
async def healthy_ports(self) -> list[int]:
now = time.monotonic()
async with self._lock:
healthy = []
for p in self.all_ports:
if self._failures[p] < self.max_failures:
healthy.append(p)
elif now - self._last_healthy[p] > self.recovery_interval:
# Give it another chance after recovery_interval.
self._failures[p] = max(0, self._failures[p] - 1)
healthy.append(p)
return healthy
async def is_healthy(self, port: int) -> bool:
ports = await self.healthy_ports
return port in ports
# ---------------------------------------------------------------------------
# Load balancer with failover
# ---------------------------------------------------------------------------
class LoadBalancer:
"""Distribute requests across healthy backend ports with auto-failover."""
def __init__(
self,
ports: list[int],
health_checker: HealthChecker,
strategy: str = "round_robin",
):
self.all_ports = ports
self.health_checker = health_checker
self.strategy = strategy
self._idx = 0
self._lock = asyncio.Lock()
def _next_round_robin(self, healthy: list[int]) -> int:
if not healthy:
return self.all_ports[self._idx % len(self.all_ports)]
for _ in range(len(self.all_ports)):
port = self.all_ports[self._idx % len(self.all_ports)]
self._idx += 1
if port in healthy:
return port
# Fallback: all unhealthy, pick first healthy anyway.
return healthy[0] if healthy else self.all_ports[0]
def _next_random(self, healthy: list[int]) -> int:
if healthy:
return random.choice(healthy)
return random.choice(self.all_ports)
async def next_port(self) -> int:
healthy = await self.health_checker.healthy_ports
async with self._lock:
if self.strategy == "round_robin":
return self._next_round_robin(healthy)
elif self.strategy in ("random", "least_conn"):
return self._next_random(healthy)
else:
return self._next_round_robin(healthy)
@property
def port_count(self) -> int:
return len(self.all_ports)
# ---------------------------------------------------------------------------
# Token helpers
# ---------------------------------------------------------------------------
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
"""Generate a random-ish prompt of approximately `token_len` tokens."""
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
"dog", "hello", "world", "deep", "seek", "model", "test",
"benchmark", "performance", "latency", "throughput", "token"]
needed = max(token_len, 1)
tokens: list[str] = []
while len(tokens) < needed:
tokens.extend(words)
return " ".join(tokens[:needed])
# ---------------------------------------------------------------------------
# aiohttp-based async request (preferred)
# ---------------------------------------------------------------------------
async def send_request_aiohttp(
session: aiohttp.ClientSession,
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
headers = {"Content-Type": "application/json"}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
first_token_received = False
last_token_time = 0.0
try:
async with session.post(url, json=payload, headers=headers) as resp:
if resp.status != 200:
text = await resp.text()
result.error = f"HTTP {resp.status}: {text[:200]}"
return result
async for line in resp.content:
line = line.decode("utf-8").strip()
if not line or not line.startswith("data: "):
continue
data_str = line[len("data: "):]
if data_str == "[DONE]":
break
try:
chunk = json.loads(data_str)
except json.JSONDecodeError:
continue
choices = chunk.get("choices", [])
if not choices:
continue
delta = choices[0].get("delta", {})
content = delta.get("content", "")
if content:
now = time.perf_counter()
if not first_token_received:
result.first_token_time_ms = (now - start_time) * 1000
first_token_received = True
else:
result.inter_token_latencies.append((now - last_token_time) * 1000)
last_token_time = now
result.output_tokens += 1
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.input_tokens = len(prompt.split()) # approximate
result.success = True
except asyncio.TimeoutError:
result.error = "timeout"
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# urllib-based synchronous request (fallback, no aiohttp)
# ---------------------------------------------------------------------------
def send_request_urllib(
host: str,
port: int,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
) -> RequestResult:
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
url = f"http://{host}:{port}/v1/chat/completions"
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": max_tokens,
"temperature": temperature,
"stream": False,
}
data = json.dumps(payload).encode("utf-8")
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
result = RequestResult(request_id=request_id, target_port=port)
start_time = time.perf_counter()
try:
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=600) as resp:
body = json.loads(resp.read().decode("utf-8"))
choices = body.get("choices", [])
if not choices:
result.error = "no choices in response"
return result
content = choices[0].get("message", {}).get("content", "")
result.output_tokens = len(content.split()) if content else 0
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
result.first_token_time_ms = result.e2e_latency_ms
result.input_tokens = len(prompt.split())
result.success = True
except Exception as e:
result.error = str(e)[:200]
return result
# ---------------------------------------------------------------------------
# Unified send_request wrapper with retry & failover
# ---------------------------------------------------------------------------
async def send_request_with_retry(
session: aiohttp.ClientSession | None,
host: str,
lb: LoadBalancer,
model: str,
request_id: int,
prompt: str,
max_tokens: int,
temperature: float,
max_retries: int = 2,
) -> RequestResult:
"""Send a request, retrying on failure with a different port each time."""
result = RequestResult(request_id=request_id)
for attempt in range(max_retries + 1):
port = await lb.next_port()
if HAS_AIOHTTP and session is not None:
result = await send_request_aiohttp(
session, host, port, model, request_id, prompt, max_tokens, temperature
)
else:
loop = asyncio.get_event_loop()
result = await loop.run_in_executor(
None, send_request_urllib,
host, port, model, request_id, prompt, max_tokens, temperature
)
result.retry_count = attempt
if result.success:
return result
# If failed, try another port on next iteration (unless last attempt).
if attempt < max_retries:
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
return result
# ---------------------------------------------------------------------------
# Benchmark orchestration
# ---------------------------------------------------------------------------
async def run_benchmark(
host: str,
ports: list[int],
lb: LoadBalancer,
health_checker: HealthChecker,
model: str,
concurrency: int,
input_len: int,
output_len: int,
num_prompts: int,
temperature: float,
output_file: str,
) -> dict[str, Any]:
"""Run the benchmark and write a JSONL file."""
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
results: list[RequestResult] = []
semaphore = asyncio.Semaphore(concurrency)
start = time.perf_counter()
if HAS_AIOHTTP:
connector = aiohttp.TCPConnector(limit=concurrency * 2)
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
await health_checker.start(session)
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
session, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
else:
await health_checker.start(None)
from concurrent.futures import ThreadPoolExecutor
async def bounded_send(idx: int, prompt: str) -> RequestResult:
async with semaphore:
return await send_request_with_retry(
None, host, lb, model, idx, prompt, output_len, temperature
)
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
results = await asyncio.gather(*tasks)
await health_checker.stop()
duration = time.perf_counter() - start
# Write JSONL
with open(output_file, "w", encoding="utf-8") as f:
for r in results:
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
# Compute summary
successful = [r for r in results if r.success]
failed = len(results) - len(successful)
retried = [r for r in results if r.retry_count > 0]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
ttfts = [r.first_token_time_ms for r in successful]
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
e2es = [r.e2e_latency_ms for r in successful]
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
total_input_tokens = sum(r.input_tokens for r in successful)
total_output_tokens = sum(r.output_tokens for r in successful)
# Per-port breakdown
per_port_stats: dict[int, dict[str, Any]] = {}
for port in ports:
port_results = [r for r in successful if r.target_port == port]
port_failed = [r for r in results if r.target_port == port and not r.success]
if port_results or port_failed:
port_ttfts = [r.first_token_time_ms for r in port_results]
port_e2es = [r.e2e_latency_ms for r in port_results]
per_port_stats[port] = {
"count": len(port_results),
"failed": len(port_failed),
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
}
# Health-check stats
healthy_at_end = await health_checker.healthy_ports
summary = {
"completed": len(successful),
"failed": failed,
"retried": len(retried),
"duration": duration,
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
"total_input_tokens": total_input_tokens,
"total_output_tokens": total_output_tokens,
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
"median_ttft_ms": percentile(ttfts, 50),
"p90_ttft_ms": percentile(ttfts, 90),
"p95_ttft_ms": percentile(ttfts, 95),
"p99_ttft_ms": percentile(ttfts, 99),
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
"median_tpot_ms": percentile(tpots, 50),
"p90_tpot_ms": percentile(tpots, 90),
"p95_tpot_ms": percentile(tpots, 95),
"p99_tpot_ms": percentile(tpots, 99),
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
"median_e2e_latency_ms": percentile(e2es, 50),
"p90_e2e_latency_ms": percentile(e2es, 90),
"p95_e2e_latency_ms": percentile(e2es, 95),
"p99_e2e_latency_ms": percentile(e2es, 99),
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
"median_itl_ms": percentile(itls, 50),
"p90_itl_ms": percentile(itls, 90),
"p95_itl_ms": percentile(itls, 95),
"p99_itl_ms": percentile(itls, 99),
"per_port_stats": per_port_stats,
"healthy_ports_at_end": healthy_at_end,
"lb_strategy": lb.strategy,
"num_services": lb.port_count,
}
# Append summary as the last line
with open(output_file, "a", encoding="utf-8") as f:
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
return summary
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=30005,
help="Single service port (backward compatible)")
parser.add_argument("--ports", default="",
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
parser.add_argument("--model", default="deepseek-v4-flash")
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--input-len", type=int, default=512)
parser.add_argument("--output-len", type=int, default=256)
parser.add_argument("--num-prompts", type=int, default=10)
parser.add_argument("--temperature", type=float, default=0.0)
parser.add_argument("--lb-strategy", default="round_robin",
choices=["round_robin", "random", "least_conn"],
help="Load balancing strategy across services")
parser.add_argument("--health-check-interval", type=float, default=5.0,
help="Seconds between health checks (default: 5)")
parser.add_argument("--health-max-failures", type=int, default=2,
help="Consecutive failures before marking a port unhealthy (default: 2)")
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
help="Seconds before retrying an unhealthy port (default: 10)")
parser.add_argument("--request-max-retries", type=int, default=2,
help="Max retries per request on failure (default: 2)")
parser.add_argument("--output-file", required=True)
args = parser.parse_args()
# Determine ports: --ports takes precedence, else fall back to --port.
if args.ports:
ports = [int(p.strip()) for p in args.ports.split(",")]
else:
ports = [args.port]
health_checker = HealthChecker(
host=args.host,
ports=ports,
check_interval=args.health_check_interval,
max_failures=args.health_max_failures,
recovery_interval=args.health_recovery_interval,
)
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
summary = asyncio.run(run_benchmark(
host=args.host,
ports=ports,
lb=lb,
health_checker=health_checker,
model=args.model,
concurrency=args.concurrency,
input_len=args.input_len,
output_len=args.output_len,
num_prompts=args.num_prompts,
temperature=args.temperature,
output_file=args.output_file,
))
print(json.dumps(summary, indent=2, ensure_ascii=False))
if __name__ == "__main__":
main()

83
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env
View File

@ -1,83 +0,0 @@
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
#
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
# Service 1: port 30005, GPUs 0,1
# Service 2: port 30006, GPUs 2,3
# Service 3: port 30007, GPUs 4,5
# Service 4: port 30008, GPUs 6,7
#
# bench_client.py distributes requests across all 4 services via round-robin
# load balancing, giving accurate cluster-wide throughput even at low
# per-service concurrency.
EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench_no_fp8"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
# Primary port (backward compatible); multi-service ports are in PORTS.
PORT="30005"
PORTS="30005"
BACKEND="vllm"
ENGINE="vllm"
# Native virtual environments on the host.
VENV_SERVER="/data/user1/yy/envs/vllm"
VENV_CLIENT="/data/user1/yy/envs/vllm"
# Load balancing strategy: round_robin | random | least_conn
LB_STRATEGY="round_robin"
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
# For single-service TP=8 we cover standard concurrency gradients.
SCENARIOS=(
"1 512 256 20"
"2 512 256 40"
"4 512 256 80"
"8 512 256 160"
"16 512 256 320"
"32 512 256 640"
"64 512 256 1280"
"128 512 256 2560"
"1 2048 512 20"
"8 2048 512 160"
"32 2048 512 640"
"128 2048 512 2560"
"1 4096 1024 20"
"8 4096 1024 160"
"32 4096 1024 640"
"128 4096 1024 2560"
"1 8192 1024 20"
"8 8192 1024 160"
"32 8192 1024 640"
"128 8192 1024 2560"
"1 16384 1024 20"
"8 16384 1024 160"
"32 16384 1024 640"
"128 16384 1024 2560"
"1 32768 1024 20"
"8 32768 1024 160"
"32 32768 1024 640"
"128 32768 1024 2560"
"1 65536 1024 20"
"8 65536 1024 160"
"32 65536 1024 640"
"128 65536 1024 2560"
"1 131072 1024 20"
"8 131072 1024 160"
"32 131072 1024 640"
"128 131072 1024 2560"
"1 262144 1024 20"
"8 262144 1024 160"
"32 262144 1024 640"
"128 262144 1024 2560"
"1 524288 1024 20"
"8 524288 1024 160"
"32 524288 1024 640"
"128 524288 1024 2560"
)
# Server start script bundled with this experiment.
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"

206
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py
View File

@ -1,206 +0,0 @@
#!/usr/bin/env python3
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
Reads JSONL files produced by bench_client.py (one request per line,
last line is the summary) and generates:
- results.json (appended scenarios, following the standard schema)
- report.md (human-readable summary)
Usage:
python3 parse_results.py <result_root>
"""
import json
import sys
from pathlib import Path
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
"""Read the summary JSON (last line) and all per-request lines."""
requests: list[dict] = []
summary: dict | None = None
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
# The summary line has "completed" and "duration" keys.
if "completed" in obj and "duration" in obj:
summary = obj
else:
requests.append(obj)
return summary, requests
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
completed = summary.get("completed", 0)
total = len(requests)
failed = total - completed if total > 0 else 0
duration_s = summary.get("duration", 0.0)
# Extract per-request metrics for percentiles.
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
def percentile(values: list[float], p: float) -> float:
if not values:
return 0.0
s = sorted(values)
k = (len(s) - 1) * p / 100.0
f = int(k)
c = min(f + 1, len(s) - 1)
return s[f] + (k - f) * (s[c] - s[f])
return {
"success": completed,
"failed": failed,
"duration_s": duration_s,
"request_throughput": summary.get("request_throughput", 0.0),
"input_token_throughput": summary.get("input_throughput", 0.0),
"output_token_throughput": summary.get("output_throughput", 0.0),
"total_token_throughput": summary.get("total_throughput", 0.0),
"total_input_tokens": summary.get("total_input_tokens", 0),
"total_output_tokens": summary.get("total_output_tokens", 0),
"e2e_ms": {
"mean": summary.get("mean_e2e_latency_ms", 0.0),
"p50": percentile(e2es, 50),
"p90": percentile(e2es, 90),
"p95": percentile(e2es, 95),
"p99": percentile(e2es, 99),
},
"ttft_ms": {
"mean": summary.get("mean_ttft_ms", 0.0),
"p50": percentile(ttfts, 50),
"p90": percentile(ttfts, 90),
"p95": percentile(ttfts, 95),
"p99": percentile(ttfts, 99),
},
"tpot_ms": {
"mean": summary.get("mean_tpot_ms", 0.0),
"p50": percentile(tpots, 50),
"p90": percentile(tpots, 90),
"p95": percentile(tpots, 95),
"p99": percentile(tpots, 99),
},
"itl_ms": {
"mean": summary.get("mean_itl_ms", 0.0),
"p50": percentile(itls, 50),
"p90": percentile(itls, 90),
"p95": percentile(itls, 95),
"p99": percentile(itls, 99),
},
}
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
return f"c{concurrency}_i{input_len}_o{output_len}"
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
if ttft_ok and tpot_ok:
mark = "✅"
elif ttft_ok or tpot_ok:
mark = "⚠️"
else:
mark = "❌"
return {
"ttft_p95_ok": ttft_ok,
"tpot_mean_ok": tpot_ok,
"overall": mark,
}
def append_scenario(results_json: Path, scenario: dict) -> None:
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"].append(scenario)
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
report_path = result_root / "report.md"
with open(report_path, "w", encoding="utf-8") as f:
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
f.write(f"- **Result root**: `{result_root}`\n\n")
f.write("## Results\n\n")
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
for s in scenarios:
cfg = s["config"]
m = s["metrics"]
slo = s.get("slo_status", {}).get("overall", "")
f.write(
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['total_token_throughput']:.2f} | "
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
)
f.write("\n")
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
def main() -> None:
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
raw_dir = result_root / "raw_outputs"
results_json = result_root / "results.json"
if not raw_dir.exists():
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
scenarios = []
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
parts = jsonl_path.stem.split("_")
if len(parts) < 5:
continue
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
summary, requests = parse_jsonl(jsonl_path)
if summary is None:
continue
metrics = compute_metrics(summary, requests)
scenario = {
"name": scenario_name(concurrency, input_len, output_len),
"config": {
"concurrency": concurrency,
"input_len": input_len,
"output_len": output_len,
"dataset": "random",
"num_prompts": metrics["success"] + metrics["failed"],
},
"metrics": metrics,
"slo_status": slo_status(metrics),
"raw_file": str(jsonl_path),
}
scenarios.append(scenario)
if not scenarios:
print("No benchmark outputs found to parse")
return
if results_json.exists():
for s in scenarios:
append_scenario(results_json, s)
generate_report(result_root, scenarios)
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
if __name__ == "__main__":
main()

272
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh
View File

@ -1,272 +0,0 @@
#!/usr/bin/env bash
# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200.
#
# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7).
# bench_client.py sends requests to the single service.
#
# Usage:
# bash run_bench.sh
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
RAW_DIR="${RESULT_ROOT}/raw_outputs"
LOG_DIR="${RESULT_ROOT}/logs"
ensure_result_root "$RESULT_ROOT"
log_init "${LOG_DIR}/orchestrator.log"
log "experiment=${EXPERIMENT_NAME}"
log "run_id=${RUN_ID}"
log "result_root=${RESULT_ROOT}"
log "platform=${PLATFORM}"
log "chip=${CHIP}"
log "accelerator=${ACCELERATOR}"
log "engine=${ENGINE}"
log "hardware=${HARDWARE}"
log "model=${MODEL_PATH}"
log "services=${PORTS}"
log "lb_strategy=${LB_STRATEGY}"
log "server_start_script=${SERVER_START_SCRIPT}"
# Write initial metadata for this run.
METADATA_JSON="${RESULT_ROOT}/results.json"
write_metadata_json \
"$METADATA_JSON" \
"$EXPERIMENT_NAME" \
"$RUN_ID" \
"$MODEL_PATH" \
"$BACKEND" \
"$ENGINE" \
"$HARDWARE" \
"$ACCELERATOR" \
"$CHIP" \
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
"$VENV_SERVER" \
"H200 1x vLLM TP=8 single-service (no FP8 KV cache) benchmark using bench_client.py"
# Update metadata with config.
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
import json, sys
path = sys.argv[1]
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
data["config"] = {
"tp": 8,
"num_services": 1,
"ports": "30005",
"gpu_groups": "0,1,2,3,4,5,6,7",
"kv_cache_dtype": "fp8",
"block_size": 256,
"max_num_seqs": 256,
"client": "bench_client.py",
"lb_strategy": "round_robin",
"scenarios": [
"1 512 256 20", "2 512 256 40", "4 512 256 80",
"8 512 256 160", "16 512 256 320", "32 512 256 640",
"64 512 256 1280", "128 512 256 2560",
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
]
}
with open(path, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
PY
# ---------------------------------------------------------------------------
# Server helpers (single-service)
# ---------------------------------------------------------------------------
is_server_healthy() {
local port="$1"
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
}
are_all_services_healthy() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
for p in "${PORT_LIST[@]}"; do
if ! is_server_healthy "$p"; then
return 1
fi
done
return 0
}
stop_all_servers() {
log "stopping all vllm services"
# Kill by PID files first.
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
[[ -f "$pid_file" ]] || continue
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "killing pid=${pid}"
kill "$pid" 2>/dev/null || true
sleep 2
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
done
# Fallback: kill any remaining vllm processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
}
start_all_servers() {
log "starting all vllm services with ${SERVER_START_SCRIPT}"
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
exit 1
fi
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
if ! are_all_services_healthy "$PORTS"; then
log "error: not all vllm services became healthy"
exit 1
fi
log "all vllm services are healthy"
}
# ---------------------------------------------------------------------------
# Pre-benchmark health check with detailed reporting
# ---------------------------------------------------------------------------
check_services_detailed() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy_count=0
local unhealthy_ports=""
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
((healthy_count++))
else
unhealthy_ports="${unhealthy_ports}${p} "
fi
done
echo "$healthy_count"
if [[ -n "$unhealthy_ports" ]]; then
echo "unhealthy: ${unhealthy_ports}" >&2
fi
}
wait_for_all_services() {
local ports="$1"
local max_wait="${2:-240}"
local interval="${3:-5}"
local elapsed=0
while (( elapsed < max_wait )); do
local healthy_count
healthy_count="$(check_services_detailed "$ports" | head -1)"
IFS=',' read -ra PORT_LIST <<< "$ports"
if (( healthy_count == ${#PORT_LIST[@]} )); then
return 0
fi
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
sleep "$interval"
((elapsed += interval))
done
return 1
}
# ---------------------------------------------------------------------------
# Mid-benchmark health check (called between scenarios)
# ---------------------------------------------------------------------------
check_and_report_services() {
local ports="$1"
IFS=',' read -ra PORT_LIST <<< "$ports"
local healthy=()
local unhealthy=()
for p in "${PORT_LIST[@]}"; do
if is_server_healthy "$p"; then
healthy+=("$p")
else
unhealthy+=("$p")
fi
done
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
if [[ ${#unhealthy[@]} -gt 0 ]]; then
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
fi
return ${#unhealthy[@]}
}
on_exit() {
local code=$?
log "orchestrator exiting with code=${code}"
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
stop_all_servers
fi
exit "$code"
}
trap on_exit EXIT
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
if ! wait_for_all_services "$PORTS"; then
log "error: not all healthy services found on ports ${PORTS}"
exit 1
fi
else
stop_all_servers
start_all_servers
fi
log "===== BENCHMARK START ====="
for scenario in "${SCENARIOS[@]}"; do
read -r concurrency input_len output_len num_prompts <<< "$scenario"
# Fallback for legacy 3-field scenarios.
if [[ -z "${num_prompts:-}" ]]; then
num_prompts="$NUM_PROMPTS"
fi
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
--host 127.0.0.1 \
--port "$PORT" \
--model "$SERVED_MODEL_NAME" \
--concurrency "$concurrency" \
--input-len "$input_len" \
--output-len "$output_len" \
--num-prompts "$num_prompts" \
--output-file "$output_file" \
> "$detail_log" 2>&1 || {
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
# After failure, check service health before continuing.
check_and_report_services "$PORTS"
continue
}
log "finished scenario: output=${output_file}"
# Between scenarios, report service health so we can spot degradation early.
check_and_report_services "$PORTS" || true
done
log "===== BENCHMARK DONE ====="
log "parsing results"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
}
log "all results saved to ${RESULT_ROOT}"

106
experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh
View File

@ -1,106 +0,0 @@
#!/bin/bash
# Start 1 vLLM TP=8 service on 8x H200.
# Uses all 8 GPUs: (0,1,2,3,4,5,6,7).
# This maximizes single-service throughput for highest-concurrency workloads.
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
VENV="${VENV_SERVER}"
export PATH="$VENV/bin:$PATH"
VLLM="$VENV/bin/vllm"
TP=8
LOG_DIR="${SCRIPT_DIR}/logs"
mkdir -p "$LOG_DIR"
# 1 service on all 8 GPUs.
# Port and GPU mapping.
SERVICES=(
"30005:0,1,2,3,4,5,6,7"
)
# Kill any existing vLLM processes for this model.
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
sleep 2
# Remove old PID files.
rm -f "${SCRIPT_DIR}"/*.pid
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
GPUS="${svc##*:}"
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
echo "Log: ${LOG}"
CUDA_VISIBLE_DEVICES="${GPUS}" \
nohup "$VLLM" serve "$MODEL_PATH" \
--trust-remote-code \
--tensor-parallel-size "$TP" \
--block-size 256 \
--served-model-name "$SERVED_MODEL_NAME" \
--port "$PORT" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
done
echo ""
echo "Waiting for service to become healthy..."
MAX_WAIT=240
all_healthy=0
for i in $(seq 1 $MAX_WAIT); do
all_healthy=1
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
all_healthy=0
break
fi
done
if [[ "$all_healthy" -eq 1 ]]; then
echo "Service is healthy!"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
echo " - http://127.0.0.1:${PORT}"
done
exit 0
fi
# Check if any process died early.
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
if [[ -f "$PID_FILE" ]]; then
PID="$(cat "$PID_FILE")"
if ! kill -0 "$PID" 2>/dev/null; then
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
tail -200 $LOG 2>/dev/null || true
exit 1
fi
fi
done
if (( i % 10 == 0 )); then
echo "Waiting... (${i}/${MAX_WAIT})"
fi
sleep 5
done
echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries"
for svc in "${SERVICES[@]}"; do
PORT="${svc%%:*}"
LOG="${LOG_DIR}/server_port${PORT}_*.log"
echo "--- Last 50 lines of port ${PORT} ---"
tail -50 $LOG 2>/dev/null || true
done
exit 1