Add master orchestrator script for sequential benchmark execution
- Implemented a bash script to run six benchmark experiments sequentially. - Added features for fault tolerance, out-of-memory recovery, and checkpoint management. - Included detailed logging for monitoring experiment progress and results. - Integrated GPU memory cleanup checks before each experiment. - Provided functionality to resume experiments from checkpoints.
This commit is contained in:
Quantong Qiu
parent
302d1a9822
commit
372ff08eea
@ -228,16 +228,22 @@ class LoadBalancer:
|
|||||||
# Token helpers
|
# Token helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
import uuid
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
def make_prompt(token_len: int) -> str:
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
prefix = str(uuid.uuid4())
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
words = [
|
||||||
needed = max(token_len, 1)
|
"the", "quick", "brown", "fox",
|
||||||
tokens: list[str] = []
|
"jumps", "over", "lazy", "dog",
|
||||||
while len(tokens) < needed:
|
"benchmark", "performance", "latency"
|
||||||
|
]
|
||||||
|
|
||||||
|
tokens = [prefix]
|
||||||
|
|
||||||
|
while len(tokens) < token_len:
|
||||||
tokens.extend(words)
|
tokens.extend(words)
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
return " ".join(tokens[:token_len])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@ -440,7 +446,36 @@ async def run_benchmark(
|
|||||||
return await send_request_with_retry(
|
return await send_request_with_retry(
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
session, host, lb, model, idx, prompt, output_len, temperature
|
||||||
)
|
)
|
||||||
|
print("Starting warmup...")
|
||||||
|
warmup_per_service = 10
|
||||||
|
warmup_requests = warmup_per_service * len(ports)
|
||||||
|
|
||||||
|
warmup_prompts = [
|
||||||
|
make_prompt(input_len)
|
||||||
|
for _ in range(warmup_requests)
|
||||||
|
]
|
||||||
|
|
||||||
|
warmup_tasks = [
|
||||||
|
send_request_with_retry(
|
||||||
|
session=session,
|
||||||
|
host=host,
|
||||||
|
lb=lb,
|
||||||
|
model=model,
|
||||||
|
request_id=-1,
|
||||||
|
prompt=p,
|
||||||
|
max_tokens=output_len,
|
||||||
|
temperature=temperature,
|
||||||
|
)
|
||||||
|
for p in warmup_prompts
|
||||||
|
]
|
||||||
|
|
||||||
|
await asyncio.gather(*warmup_tasks)
|
||||||
|
|
||||||
|
print("Warmup finished.")
|
||||||
|
|
||||||
|
# 给 CUDA / Scheduler 一点时间稳定
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
await health_checker.stop()
|
await health_checker.stop()
|
||||||
|
|||||||
@ -34,50 +34,146 @@ LB_STRATEGY="round_robin"
|
|||||||
# For multi-service we include low-concurrency scenarios to show how
|
# For multi-service we include low-concurrency scenarios to show how
|
||||||
# requests spread across 4 services.
|
# requests spread across 4 services.
|
||||||
SCENARIOS=(
|
SCENARIOS=(
|
||||||
"1 512 256 20"
|
"1 512 256 20"
|
||||||
"2 512 256 40"
|
"2 512 256 40"
|
||||||
"4 512 256 80"
|
"4 512 256 80"
|
||||||
"8 512 256 160"
|
"8 512 256 160"
|
||||||
"16 512 256 320"
|
"16 512 256 320"
|
||||||
"32 512 256 640"
|
"32 512 256 640"
|
||||||
"64 512 256 1280"
|
"64 512 256 1280"
|
||||||
"128 512 256 2560"
|
"128 512 256 2560"
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
"1 1024 128 20"
|
||||||
"32 2048 512 640"
|
"1 1024 256 20"
|
||||||
"128 2048 512 2560"
|
"1 1024 512 20"
|
||||||
"1 4096 1024 20"
|
"1 1024 1024 20"
|
||||||
"8 4096 1024 160"
|
"1 1024 2048 20"
|
||||||
"32 4096 1024 640"
|
"1 1024 4096 20"
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
"8 1024 128 160"
|
||||||
"8 8192 1024 160"
|
"8 1024 256 160"
|
||||||
"32 8192 1024 640"
|
"8 1024 512 160"
|
||||||
"128 8192 1024 2560"
|
"8 1024 1024 160"
|
||||||
"1 16384 1024 20"
|
"8 1024 2048 160"
|
||||||
"8 16384 1024 160"
|
"8 1024 4096 160"
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
"32 1024 128 640"
|
||||||
"1 32768 1024 20"
|
"32 1024 256 640"
|
||||||
"8 32768 1024 160"
|
"32 1024 512 640"
|
||||||
"32 32768 1024 640"
|
"32 1024 1024 640"
|
||||||
"128 32768 1024 2560"
|
"32 1024 2048 640"
|
||||||
"1 65536 1024 20"
|
"32 1024 4096 640"
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
"128 1024 128 2560"
|
||||||
"128 65536 1024 2560"
|
"128 1024 256 2560"
|
||||||
"1 131072 1024 20"
|
"128 1024 512 2560"
|
||||||
"8 131072 1024 160"
|
"128 1024 1024 2560"
|
||||||
"32 131072 1024 640"
|
"128 1024 2048 2560"
|
||||||
"128 131072 1024 2560"
|
"128 1024 4096 2560"
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
"1 2048 128 20"
|
||||||
"32 262144 1024 640"
|
"1 2048 256 20"
|
||||||
"128 262144 1024 2560"
|
"1 2048 512 20"
|
||||||
"1 524288 1024 20"
|
"1 2048 1024 20"
|
||||||
"8 524288 1024 160"
|
"1 2048 2048 20"
|
||||||
"32 524288 1024 640"
|
"1 2048 4096 20"
|
||||||
"128 524288 1024 2560"
|
|
||||||
|
"8 2048 128 160"
|
||||||
|
"8 2048 256 160"
|
||||||
|
"8 2048 512 160"
|
||||||
|
"8 2048 1024 160"
|
||||||
|
"8 2048 2048 160"
|
||||||
|
"8 2048 4096 160"
|
||||||
|
|
||||||
|
"32 2048 128 640"
|
||||||
|
"32 2048 256 640"
|
||||||
|
"32 2048 512 640"
|
||||||
|
"32 2048 1024 640"
|
||||||
|
"32 2048 2048 640"
|
||||||
|
"32 2048 4096 640"
|
||||||
|
|
||||||
|
"128 2048 128 2560"
|
||||||
|
"128 2048 256 2560"
|
||||||
|
"128 2048 512 2560"
|
||||||
|
"128 2048 1024 2560"
|
||||||
|
"128 2048 2048 2560"
|
||||||
|
"128 2048 4096 2560"
|
||||||
|
|
||||||
|
"1 4096 128 20"
|
||||||
|
"1 4096 256 20"
|
||||||
|
"1 4096 512 20"
|
||||||
|
"1 4096 1024 20"
|
||||||
|
"1 4096 2048 20"
|
||||||
|
"1 4096 4096 20"
|
||||||
|
|
||||||
|
"8 4096 128 160"
|
||||||
|
"8 4096 256 160"
|
||||||
|
"8 4096 512 160"
|
||||||
|
"8 4096 1024 160"
|
||||||
|
"8 4096 2048 160"
|
||||||
|
"8 4096 4096 160"
|
||||||
|
|
||||||
|
"32 4096 128 640"
|
||||||
|
"32 4096 256 640"
|
||||||
|
"32 4096 512 640"
|
||||||
|
"32 4096 1024 640"
|
||||||
|
"32 4096 2048 640"
|
||||||
|
"32 4096 4096 640"
|
||||||
|
|
||||||
|
"128 4096 128 2560"
|
||||||
|
"128 4096 256 2560"
|
||||||
|
"128 4096 512 2560"
|
||||||
|
"128 4096 1024 2560"
|
||||||
|
"128 4096 2048 2560"
|
||||||
|
"128 4096 4096 2560"
|
||||||
|
|
||||||
|
"1 16384 128 20"
|
||||||
|
"1 16384 256 20"
|
||||||
|
"1 16384 512 20"
|
||||||
|
"1 16384 1024 20"
|
||||||
|
"1 16384 2048 20"
|
||||||
|
|
||||||
|
"8 16384 128 160"
|
||||||
|
"8 16384 256 160"
|
||||||
|
"8 16384 512 160"
|
||||||
|
"8 16384 1024 160"
|
||||||
|
"8 16384 2048 160"
|
||||||
|
|
||||||
|
"32 16384 128 640"
|
||||||
|
"32 16384 256 640"
|
||||||
|
"32 16384 512 640"
|
||||||
|
"32 16384 1024 640"
|
||||||
|
"32 16384 2048 640"
|
||||||
|
|
||||||
|
"128 16384 128 2560"
|
||||||
|
"128 16384 256 2560"
|
||||||
|
"128 16384 512 2560"
|
||||||
|
"128 16384 1024 2560"
|
||||||
|
"128 16384 2048 2560"
|
||||||
|
|
||||||
|
|
||||||
|
"1 65536 128 20"
|
||||||
|
"1 65536 256 20"
|
||||||
|
"1 65536 512 20"
|
||||||
|
"1 65536 1024 20"
|
||||||
|
|
||||||
|
"8 65536 128 160"
|
||||||
|
"8 65536 256 160"
|
||||||
|
"8 65536 512 160"
|
||||||
|
"8 65536 1024 160"
|
||||||
|
"32 65536 1024 160"
|
||||||
|
|
||||||
|
"1 131072 128 20"
|
||||||
|
"1 131072 256 20"
|
||||||
|
"1 131072 512 20"
|
||||||
|
|
||||||
|
"8 131072 512 160"
|
||||||
|
|
||||||
|
"1 262144 256 20"
|
||||||
|
|
||||||
|
"8 262144 128 160"
|
||||||
|
|
||||||
|
"1 524288 128 20"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
# Server start script bundled with this experiment.
|
||||||
|
|||||||
@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
|
||||||
"run_id": "20260710-091522",
|
|
||||||
"timestamp": "2026-07-10T09:15:22+00:00",
|
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
|
||||||
"backend": "vllm",
|
|
||||||
"engine": "vllm",
|
|
||||||
"hardware": "8x NVIDIA H200 143GB",
|
|
||||||
"accelerator": "NVIDIA H200",
|
|
||||||
"chip": "nvidia_h200",
|
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
|
||||||
"git_commit": "39fb076",
|
|
||||||
"git_dirty": "dirty",
|
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20",
|
|
||||||
"4 512 256 80",
|
|
||||||
"8 512 256 160",
|
|
||||||
"16 512 256 320",
|
|
||||||
"32 512 256 640",
|
|
||||||
"64 512 256 1280",
|
|
||||||
"128 512 256 2560",
|
|
||||||
"1 2048 512 20",
|
|
||||||
"8 2048 512 160",
|
|
||||||
"32 2048 512 640",
|
|
||||||
"128 2048 512 2560",
|
|
||||||
"1 4096 1024 20",
|
|
||||||
"8 4096 1024 160",
|
|
||||||
"32 4096 1024 640",
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"scenarios": []
|
|
||||||
}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
|
||||||
"run_id": "20260710-091700",
|
|
||||||
"timestamp": "2026-07-10T09:17:00+00:00",
|
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
|
||||||
"backend": "vllm",
|
|
||||||
"engine": "vllm",
|
|
||||||
"hardware": "8x NVIDIA H200 143GB",
|
|
||||||
"accelerator": "NVIDIA H200",
|
|
||||||
"chip": "nvidia_h200",
|
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
|
||||||
"git_commit": "39fb076",
|
|
||||||
"git_dirty": "dirty",
|
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20",
|
|
||||||
"4 512 256 80",
|
|
||||||
"8 512 256 160",
|
|
||||||
"16 512 256 320",
|
|
||||||
"32 512 256 640",
|
|
||||||
"64 512 256 1280",
|
|
||||||
"128 512 256 2560",
|
|
||||||
"1 2048 512 20",
|
|
||||||
"8 2048 512 160",
|
|
||||||
"32 2048 512 640",
|
|
||||||
"128 2048 512 2560",
|
|
||||||
"1 4096 1024 20",
|
|
||||||
"8 4096 1024 160",
|
|
||||||
"32 4096 1024 640",
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"scenarios": []
|
|
||||||
}
|
|
||||||
@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
||||||
"run_id": "20260710-091141",
|
"run_id": "20260710-104930",
|
||||||
"timestamp": "2026-07-10T09:11:41+00:00",
|
"timestamp": "2026-07-10T10:49:30+00:00",
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
"model": "/data/models/DeepSeek-V4-Flash",
|
||||||
"backend": "vllm",
|
"backend": "vllm",
|
||||||
"engine": "vllm",
|
"engine": "vllm",
|
||||||
@ -11,7 +11,7 @@
|
|||||||
"chip": "nvidia_h200",
|
"chip": "nvidia_h200",
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
"env": "/data/user1/yy/envs/vllm",
|
||||||
"git_commit": "39fb076",
|
"git_commit": "def1355",
|
||||||
"git_dirty": "dirty",
|
"git_dirty": "dirty",
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
||||||
},
|
},
|
||||||
@ -1,273 +0,0 @@
|
|||||||
# dsv4_h200_vllm_tp2_custom_bench_no_fp8
|
|
||||||
|
|
||||||
> **Note:** This variant does **NOT** use `--kv-cache-dtype fp8`.
|
|
||||||
|
|
||||||
## 目的
|
|
||||||
|
|
||||||
当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU：
|
|
||||||
|
|
||||||
| 服务 | 端口 | GPU 对 |
|
|
||||||
|------|------|--------|
|
|
||||||
| 1 | 30005 | 0, 1 |
|
|
||||||
| 2 | 30006 | 2, 3 |
|
|
||||||
| 3 | 30007 | 4, 5 |
|
|
||||||
| 4 | 30008 | 6, 7 |
|
|
||||||
|
|
||||||
此时 `sglang.bench_serving` 存在以下问题：
|
|
||||||
|
|
||||||
1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。
|
|
||||||
2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。
|
|
||||||
3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。
|
|
||||||
|
|
||||||
因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。
|
|
||||||
|
|
||||||
## 与 sglang.bench_serving 的区别
|
|
||||||
|
|
||||||
| 特性 | sglang.bench_serving | bench_client.py (本实验) |
|
|
||||||
|------|----------------------|--------------------------|
|
|
||||||
| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) |
|
|
||||||
| 并发控制 | 进程级 | `asyncio.Semaphore` |
|
|
||||||
| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` |
|
|
||||||
| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 |
|
|
||||||
| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 |
|
|
||||||
| 低并发精度 | 一般 | 高（适合 TP=2 场景） |
|
|
||||||
| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 |
|
|
||||||
|
|
||||||
## 文件说明
|
|
||||||
|
|
||||||
| 文件 | 说明 |
|
|
||||||
|------|------|
|
|
||||||
| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 |
|
|
||||||
| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） |
|
|
||||||
| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 |
|
|
||||||
| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） |
|
|
||||||
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
|
|
||||||
|
|
||||||
## 快速运行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 完整流程（自动启动 4 个服务、压测、生成报告）
|
|
||||||
bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 跳过服务管理（已有 4 个服务在运行）
|
|
||||||
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 单独运行压测客户端（多服务，带健康检查）
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports 30005,30006,30007,30008 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
|
|
||||||
--lb-strategy round_robin \
|
|
||||||
--health-check-interval 5 \
|
|
||||||
--health-max-failures 2 \
|
|
||||||
--health-recovery-interval 10 \
|
|
||||||
--request-max-retries 2 \
|
|
||||||
--output-file /tmp/test.jsonl
|
|
||||||
|
|
||||||
# 单独运行压测客户端（单服务，向后兼容）
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \
|
|
||||||
--output-file /tmp/test_single.jsonl
|
|
||||||
|
|
||||||
# 解析已有结果
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \
|
|
||||||
experiments/dsv4_h200_vllm_tp2_custom_bench/results/<RUN_ID>
|
|
||||||
```
|
|
||||||
|
|
||||||
## 场景设计
|
|
||||||
|
|
||||||
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
|
|
||||||
|
|
||||||
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|
|
||||||
|------|----------|----------|--------|------|
|
|
||||||
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
|
|
||||||
| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req |
|
|
||||||
| 8 | 512 | 256 | 160 | 中低并发 |
|
|
||||||
| 16 | 512 | 256 | 320 | 中等并发 |
|
|
||||||
| 32 | 512 | 256 | 640 | 中高并发 |
|
|
||||||
| 64 | 512 | 256 | 1280 | 高并发 |
|
|
||||||
| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 |
|
|
||||||
| 1 | 2048 | 512 | 20 | 长输入基线 |
|
|
||||||
| 8 | 2048 | 512 | 160 | 长输入中并发 |
|
|
||||||
| 32 | 2048 | 512 | 640 | 长输入高并发 |
|
|
||||||
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
|
|
||||||
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
|
|
||||||
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
|
|
||||||
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
|
|
||||||
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
|
|
||||||
|
|
||||||
## 健康检查与故障自动跳过
|
|
||||||
|
|
||||||
`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。
|
|
||||||
|
|
||||||
### 健康检查机制
|
|
||||||
|
|
||||||
| 参数 | 默认值 | 说明 |
|
|
||||||
|------|--------|------|
|
|
||||||
| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 |
|
|
||||||
| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 |
|
|
||||||
| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 |
|
|
||||||
| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 |
|
|
||||||
|
|
||||||
### 故障处理流程
|
|
||||||
|
|
||||||
```
|
|
||||||
请求 → 负载均衡选择端口 → 发送请求
|
|
||||||
↓
|
|
||||||
成功 → 记录结果
|
|
||||||
↓
|
|
||||||
失败 → 换另一个端口重试（最多 --request-max-retries 次）
|
|
||||||
↓
|
|
||||||
全部失败 → 记录失败，继续下一个请求
|
|
||||||
```
|
|
||||||
|
|
||||||
后台同时运行健康检查：
|
|
||||||
- 每 5 秒对所有端口执行 `/health` 探测
|
|
||||||
- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求
|
|
||||||
- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池
|
|
||||||
|
|
||||||
### 输出中的故障信息
|
|
||||||
|
|
||||||
汇总 JSON 包含以下字段：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"completed": 638,
|
|
||||||
"failed": 2,
|
|
||||||
"retried": 5,
|
|
||||||
"healthy_ports_at_end": [30005, 30006, 30007],
|
|
||||||
"per_port_stats": {
|
|
||||||
"30005": {"count": 160, "failed": 0, ...},
|
|
||||||
"30006": {"count": 159, "failed": 1, ...},
|
|
||||||
"30007": {"count": 160, "failed": 0, ...},
|
|
||||||
"30008": {"count": 159, "failed": 1, ...}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `failed`: 最终失败的请求数（重试后仍失败）
|
|
||||||
- `retried`: 至少重试过一次的请求数
|
|
||||||
- `healthy_ports_at_end`: 压测结束时仍健康的服务端口
|
|
||||||
- `per_port_stats[].failed`: 每个端口的失败数
|
|
||||||
|
|
||||||
### 场景间健康检查
|
|
||||||
|
|
||||||
`run_bench.sh` 在每个场景结束后会打印一次服务健康状态：
|
|
||||||
|
|
||||||
```
|
|
||||||
[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[]
|
|
||||||
[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008]
|
|
||||||
[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008
|
|
||||||
```
|
|
||||||
|
|
||||||
这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。
|
|
||||||
|
|
||||||
### 自定义故障参数
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python3 bench_client.py \
|
|
||||||
--ports 30005,30006,30007,30008 \
|
|
||||||
--health-check-interval 3 \
|
|
||||||
--health-max-failures 1 \
|
|
||||||
--health-recovery-interval 5 \
|
|
||||||
--request-max-retries 3 \
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复
|
|
||||||
- 更容错：单个请求最多重试 3 次
|
|
||||||
|
|
||||||
## 负载均衡策略
|
|
||||||
|
|
||||||
`bench_client.py` 支持三种负载均衡策略：
|
|
||||||
|
|
||||||
| 策略 | 说明 | 适用场景 |
|
|
||||||
|------|------|----------|
|
|
||||||
| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 |
|
|
||||||
| `random` | 随机选择一个端口 | 简单，分布可能不均 |
|
|
||||||
| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 |
|
|
||||||
|
|
||||||
通过 `--lb-strategy` 参数选择：
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ...
|
|
||||||
```
|
|
||||||
|
|
||||||
## 输出格式
|
|
||||||
|
|
||||||
### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`)
|
|
||||||
|
|
||||||
每行一个 JSON 对象，包含请求分发到的目标端口：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"request_id": 0,
|
|
||||||
"input_tokens": 512,
|
|
||||||
"output_tokens": 256,
|
|
||||||
"first_token_time_ms": 45.23,
|
|
||||||
"e2e_latency_ms": 5234.56,
|
|
||||||
"inter_token_latencies": [18.5, 19.2, ...],
|
|
||||||
"tpot_ms": 19.8,
|
|
||||||
"mean_itl_ms": 19.5,
|
|
||||||
"success": true,
|
|
||||||
"error": "",
|
|
||||||
"target_port": 30005
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
最后一行是汇总数据，包含 **各端口统计**：
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"completed": 640,
|
|
||||||
"failed": 0,
|
|
||||||
"duration": 12.34,
|
|
||||||
"request_throughput": 51.88,
|
|
||||||
"mean_ttft_ms": 48.5,
|
|
||||||
"p95_ttft_ms": 62.3,
|
|
||||||
"per_port_stats": {
|
|
||||||
"30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0},
|
|
||||||
"30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0},
|
|
||||||
"30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0},
|
|
||||||
"30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0}
|
|
||||||
},
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"num_services": 4
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### 结构化结果 (`results.json`)
|
|
||||||
|
|
||||||
遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。
|
|
||||||
|
|
||||||
### 人类可读报告 (`report.md`)
|
|
||||||
|
|
||||||
Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。
|
|
||||||
|
|
||||||
## 单独启动/停止 4 个服务
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 启动 4 个服务
|
|
||||||
bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh
|
|
||||||
|
|
||||||
# 停止（run_bench.sh 会自动停止，但也可以手动）
|
|
||||||
for port in 30005 30006 30007 30008; do
|
|
||||||
pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid"
|
|
||||||
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
|
|
||||||
rm -f "$pid_file"
|
|
||||||
done
|
|
||||||
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。
|
|
||||||
- 默认 `temperature=0.0` 以保证输出长度稳定。
|
|
||||||
- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。
|
|
||||||
- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。
|
|
||||||
- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。
|
|
||||||
- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。
|
|
||||||
- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。
|
|
||||||
@ -1,617 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
|
|
||||||
|
|
||||||
Sends concurrent requests to multiple inference servers and records per-request
|
|
||||||
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
|
|
||||||
|
|
||||||
Designed for TP=2 on 8-GPU setups where 4 independent services run on
|
|
||||||
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
|
|
||||||
round-robin or random load balancing. Unhealthy services are automatically
|
|
||||||
skipped with periodic retry.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# Single service (backward compatible)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
# Multi-service (4 services on ports 30005-30008)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports 30005,30006,30007,30008 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--lb-strategy round_robin \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
The output JSONL contains one object per request with raw timing data.
|
|
||||||
A final summary line ("completed" field) aggregates all requests.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import random
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from dataclasses import asdict, dataclass, field
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
# Try aiohttp first; fall back to urllib for stdlib-only environments.
|
|
||||||
try:
|
|
||||||
import aiohttp
|
|
||||||
HAS_AIOHTTP = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_AIOHTTP = False
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Data structures
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class RequestResult:
|
|
||||||
request_id: int
|
|
||||||
input_tokens: int = 0
|
|
||||||
output_tokens: int = 0
|
|
||||||
first_token_time_ms: float = 0.0 # TTFT
|
|
||||||
e2e_latency_ms: float = 0.0 # total wall time
|
|
||||||
inter_token_latencies: list[float] = field(default_factory=list)
|
|
||||||
success: bool = False
|
|
||||||
error: str = ""
|
|
||||||
target_port: int = 0 # which service handled this request
|
|
||||||
retry_count: int = 0 # how many times this request was retried
|
|
||||||
|
|
||||||
@property
|
|
||||||
def tpot_ms(self) -> float:
|
|
||||||
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
|
|
||||||
if self.output_tokens <= 1:
|
|
||||||
return 0.0
|
|
||||||
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def mean_itl_ms(self) -> float:
|
|
||||||
if not self.inter_token_latencies:
|
|
||||||
return 0.0
|
|
||||||
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
|
|
||||||
|
|
||||||
def to_dict(self) -> dict[str, Any]:
|
|
||||||
d = asdict(self)
|
|
||||||
d["tpot_ms"] = self.tpot_ms
|
|
||||||
d["mean_itl_ms"] = self.mean_itl_ms
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Health checker
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class HealthChecker:
|
|
||||||
"""Periodically check service health and maintain a healthy-port list."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
check_interval: float = 5.0,
|
|
||||||
max_failures: int = 2,
|
|
||||||
recovery_interval: float = 10.0,
|
|
||||||
):
|
|
||||||
self.host = host
|
|
||||||
self.all_ports = ports
|
|
||||||
self.check_interval = check_interval
|
|
||||||
self.max_failures = max_failures
|
|
||||||
self.recovery_interval = recovery_interval
|
|
||||||
|
|
||||||
# port -> consecutive failure count
|
|
||||||
self._failures: dict[int, int] = {p: 0 for p in ports}
|
|
||||||
# port -> last healthy timestamp
|
|
||||||
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
self._task: asyncio.Task | None = None
|
|
||||||
|
|
||||||
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
|
|
||||||
url = f"http://{self.host}:{port}/health"
|
|
||||||
try:
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
else:
|
|
||||||
req = urllib.request.Request(url, method="GET")
|
|
||||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
checks = [self._check_one(p, session) for p in self.all_ports]
|
|
||||||
results = await asyncio.gather(*checks, return_exceptions=True)
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
for port, ok in zip(self.all_ports, results):
|
|
||||||
if isinstance(ok, Exception):
|
|
||||||
ok = False
|
|
||||||
if ok:
|
|
||||||
self._failures[port] = 0
|
|
||||||
self._last_healthy[port] = now
|
|
||||||
else:
|
|
||||||
self._failures[port] += 1
|
|
||||||
|
|
||||||
async def start(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
async def _loop() -> None:
|
|
||||||
while True:
|
|
||||||
await asyncio.sleep(self.check_interval)
|
|
||||||
await self._check_all(session)
|
|
||||||
self._task = asyncio.create_task(_loop())
|
|
||||||
|
|
||||||
async def stop(self) -> None:
|
|
||||||
if self._task is not None:
|
|
||||||
self._task.cancel()
|
|
||||||
try:
|
|
||||||
await self._task
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
async def healthy_ports(self) -> list[int]:
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
healthy = []
|
|
||||||
for p in self.all_ports:
|
|
||||||
if self._failures[p] < self.max_failures:
|
|
||||||
healthy.append(p)
|
|
||||||
elif now - self._last_healthy[p] > self.recovery_interval:
|
|
||||||
# Give it another chance after recovery_interval.
|
|
||||||
self._failures[p] = max(0, self._failures[p] - 1)
|
|
||||||
healthy.append(p)
|
|
||||||
return healthy
|
|
||||||
|
|
||||||
async def is_healthy(self, port: int) -> bool:
|
|
||||||
ports = await self.healthy_ports
|
|
||||||
return port in ports
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Load balancer with failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class LoadBalancer:
|
|
||||||
"""Distribute requests across healthy backend ports with auto-failover."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
ports: list[int],
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
strategy: str = "round_robin",
|
|
||||||
):
|
|
||||||
self.all_ports = ports
|
|
||||||
self.health_checker = health_checker
|
|
||||||
self.strategy = strategy
|
|
||||||
self._idx = 0
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
|
|
||||||
def _next_round_robin(self, healthy: list[int]) -> int:
|
|
||||||
if not healthy:
|
|
||||||
return self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
for _ in range(len(self.all_ports)):
|
|
||||||
port = self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
self._idx += 1
|
|
||||||
if port in healthy:
|
|
||||||
return port
|
|
||||||
# Fallback: all unhealthy, pick first healthy anyway.
|
|
||||||
return healthy[0] if healthy else self.all_ports[0]
|
|
||||||
|
|
||||||
def _next_random(self, healthy: list[int]) -> int:
|
|
||||||
if healthy:
|
|
||||||
return random.choice(healthy)
|
|
||||||
return random.choice(self.all_ports)
|
|
||||||
|
|
||||||
async def next_port(self) -> int:
|
|
||||||
healthy = await self.health_checker.healthy_ports
|
|
||||||
async with self._lock:
|
|
||||||
if self.strategy == "round_robin":
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
elif self.strategy in ("random", "least_conn"):
|
|
||||||
return self._next_random(healthy)
|
|
||||||
else:
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def port_count(self) -> int:
|
|
||||||
return len(self.all_ports)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Token helpers
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
|
||||||
needed = max(token_len, 1)
|
|
||||||
tokens: list[str] = []
|
|
||||||
while len(tokens) < needed:
|
|
||||||
tokens.extend(words)
|
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# aiohttp-based async request (preferred)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_aiohttp(
|
|
||||||
session: aiohttp.ClientSession,
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": True,
|
|
||||||
}
|
|
||||||
headers = {"Content-Type": "application/json"}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
first_token_received = False
|
|
||||||
last_token_time = 0.0
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with session.post(url, json=payload, headers=headers) as resp:
|
|
||||||
if resp.status != 200:
|
|
||||||
text = await resp.text()
|
|
||||||
result.error = f"HTTP {resp.status}: {text[:200]}"
|
|
||||||
return result
|
|
||||||
|
|
||||||
async for line in resp.content:
|
|
||||||
line = line.decode("utf-8").strip()
|
|
||||||
if not line or not line.startswith("data: "):
|
|
||||||
continue
|
|
||||||
data_str = line[len("data: "):]
|
|
||||||
if data_str == "[DONE]":
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
chunk = json.loads(data_str)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
choices = chunk.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
continue
|
|
||||||
|
|
||||||
delta = choices[0].get("delta", {})
|
|
||||||
content = delta.get("content", "")
|
|
||||||
if content:
|
|
||||||
now = time.perf_counter()
|
|
||||||
if not first_token_received:
|
|
||||||
result.first_token_time_ms = (now - start_time) * 1000
|
|
||||||
first_token_received = True
|
|
||||||
else:
|
|
||||||
result.inter_token_latencies.append((now - last_token_time) * 1000)
|
|
||||||
last_token_time = now
|
|
||||||
result.output_tokens += 1
|
|
||||||
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.input_tokens = len(prompt.split()) # approximate
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
result.error = "timeout"
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# urllib-based synchronous request (fallback, no aiohttp)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def send_request_urllib(
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": False,
|
|
||||||
}
|
|
||||||
data = json.dumps(payload).encode("utf-8")
|
|
||||||
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
|
|
||||||
try:
|
|
||||||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
|
||||||
with urllib.request.urlopen(req, timeout=600) as resp:
|
|
||||||
body = json.loads(resp.read().decode("utf-8"))
|
|
||||||
|
|
||||||
choices = body.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
result.error = "no choices in response"
|
|
||||||
return result
|
|
||||||
|
|
||||||
content = choices[0].get("message", {}).get("content", "")
|
|
||||||
result.output_tokens = len(content.split()) if content else 0
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.first_token_time_ms = result.e2e_latency_ms
|
|
||||||
result.input_tokens = len(prompt.split())
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Unified send_request wrapper with retry & failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_with_retry(
|
|
||||||
session: aiohttp.ClientSession | None,
|
|
||||||
host: str,
|
|
||||||
lb: LoadBalancer,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
max_retries: int = 2,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Send a request, retrying on failure with a different port each time."""
|
|
||||||
result = RequestResult(request_id=request_id)
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
port = await lb.next_port()
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
result = await send_request_aiohttp(
|
|
||||||
session, host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
result = await loop.run_in_executor(
|
|
||||||
None, send_request_urllib,
|
|
||||||
host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
result.retry_count = attempt
|
|
||||||
if result.success:
|
|
||||||
return result
|
|
||||||
# If failed, try another port on next iteration (unless last attempt).
|
|
||||||
if attempt < max_retries:
|
|
||||||
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Benchmark orchestration
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def run_benchmark(
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
lb: LoadBalancer,
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
model: str,
|
|
||||||
concurrency: int,
|
|
||||||
input_len: int,
|
|
||||||
output_len: int,
|
|
||||||
num_prompts: int,
|
|
||||||
temperature: float,
|
|
||||||
output_file: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Run the benchmark and write a JSONL file."""
|
|
||||||
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
|
|
||||||
results: list[RequestResult] = []
|
|
||||||
semaphore = asyncio.Semaphore(concurrency)
|
|
||||||
|
|
||||||
start = time.perf_counter()
|
|
||||||
|
|
||||||
if HAS_AIOHTTP:
|
|
||||||
connector = aiohttp.TCPConnector(limit=concurrency * 2)
|
|
||||||
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
|
|
||||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
|
||||||
await health_checker.start(session)
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
else:
|
|
||||||
await health_checker.start(None)
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
None, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
|
|
||||||
duration = time.perf_counter() - start
|
|
||||||
|
|
||||||
# Write JSONL
|
|
||||||
with open(output_file, "w", encoding="utf-8") as f:
|
|
||||||
for r in results:
|
|
||||||
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
# Compute summary
|
|
||||||
successful = [r for r in results if r.success]
|
|
||||||
failed = len(results) - len(successful)
|
|
||||||
retried = [r for r in results if r.retry_count > 0]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
ttfts = [r.first_token_time_ms for r in successful]
|
|
||||||
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
|
|
||||||
e2es = [r.e2e_latency_ms for r in successful]
|
|
||||||
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
|
|
||||||
|
|
||||||
total_input_tokens = sum(r.input_tokens for r in successful)
|
|
||||||
total_output_tokens = sum(r.output_tokens for r in successful)
|
|
||||||
|
|
||||||
# Per-port breakdown
|
|
||||||
per_port_stats: dict[int, dict[str, Any]] = {}
|
|
||||||
for port in ports:
|
|
||||||
port_results = [r for r in successful if r.target_port == port]
|
|
||||||
port_failed = [r for r in results if r.target_port == port and not r.success]
|
|
||||||
if port_results or port_failed:
|
|
||||||
port_ttfts = [r.first_token_time_ms for r in port_results]
|
|
||||||
port_e2es = [r.e2e_latency_ms for r in port_results]
|
|
||||||
per_port_stats[port] = {
|
|
||||||
"count": len(port_results),
|
|
||||||
"failed": len(port_failed),
|
|
||||||
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
|
|
||||||
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
|
|
||||||
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Health-check stats
|
|
||||||
healthy_at_end = await health_checker.healthy_ports
|
|
||||||
|
|
||||||
summary = {
|
|
||||||
"completed": len(successful),
|
|
||||||
"failed": failed,
|
|
||||||
"retried": len(retried),
|
|
||||||
"duration": duration,
|
|
||||||
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
|
|
||||||
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
|
|
||||||
"total_input_tokens": total_input_tokens,
|
|
||||||
"total_output_tokens": total_output_tokens,
|
|
||||||
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
|
|
||||||
"median_ttft_ms": percentile(ttfts, 50),
|
|
||||||
"p90_ttft_ms": percentile(ttfts, 90),
|
|
||||||
"p95_ttft_ms": percentile(ttfts, 95),
|
|
||||||
"p99_ttft_ms": percentile(ttfts, 99),
|
|
||||||
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
|
|
||||||
"median_tpot_ms": percentile(tpots, 50),
|
|
||||||
"p90_tpot_ms": percentile(tpots, 90),
|
|
||||||
"p95_tpot_ms": percentile(tpots, 95),
|
|
||||||
"p99_tpot_ms": percentile(tpots, 99),
|
|
||||||
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
|
|
||||||
"median_e2e_latency_ms": percentile(e2es, 50),
|
|
||||||
"p90_e2e_latency_ms": percentile(e2es, 90),
|
|
||||||
"p95_e2e_latency_ms": percentile(e2es, 95),
|
|
||||||
"p99_e2e_latency_ms": percentile(e2es, 99),
|
|
||||||
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
|
|
||||||
"median_itl_ms": percentile(itls, 50),
|
|
||||||
"p90_itl_ms": percentile(itls, 90),
|
|
||||||
"p95_itl_ms": percentile(itls, 95),
|
|
||||||
"p99_itl_ms": percentile(itls, 99),
|
|
||||||
"per_port_stats": per_port_stats,
|
|
||||||
"healthy_ports_at_end": healthy_at_end,
|
|
||||||
"lb_strategy": lb.strategy,
|
|
||||||
"num_services": lb.port_count,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Append summary as the last line
|
|
||||||
with open(output_file, "a", encoding="utf-8") as f:
|
|
||||||
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
|
|
||||||
parser.add_argument("--host", default="127.0.0.1")
|
|
||||||
parser.add_argument("--port", type=int, default=30005,
|
|
||||||
help="Single service port (backward compatible)")
|
|
||||||
parser.add_argument("--ports", default="",
|
|
||||||
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
|
|
||||||
parser.add_argument("--model", default="deepseek-v4-flash")
|
|
||||||
parser.add_argument("--concurrency", type=int, default=1)
|
|
||||||
parser.add_argument("--input-len", type=int, default=512)
|
|
||||||
parser.add_argument("--output-len", type=int, default=256)
|
|
||||||
parser.add_argument("--num-prompts", type=int, default=10)
|
|
||||||
parser.add_argument("--temperature", type=float, default=0.0)
|
|
||||||
parser.add_argument("--lb-strategy", default="round_robin",
|
|
||||||
choices=["round_robin", "random", "least_conn"],
|
|
||||||
help="Load balancing strategy across services")
|
|
||||||
parser.add_argument("--health-check-interval", type=float, default=5.0,
|
|
||||||
help="Seconds between health checks (default: 5)")
|
|
||||||
parser.add_argument("--health-max-failures", type=int, default=2,
|
|
||||||
help="Consecutive failures before marking a port unhealthy (default: 2)")
|
|
||||||
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
|
|
||||||
help="Seconds before retrying an unhealthy port (default: 10)")
|
|
||||||
parser.add_argument("--request-max-retries", type=int, default=2,
|
|
||||||
help="Max retries per request on failure (default: 2)")
|
|
||||||
parser.add_argument("--output-file", required=True)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Determine ports: --ports takes precedence, else fall back to --port.
|
|
||||||
if args.ports:
|
|
||||||
ports = [int(p.strip()) for p in args.ports.split(",")]
|
|
||||||
else:
|
|
||||||
ports = [args.port]
|
|
||||||
|
|
||||||
health_checker = HealthChecker(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
check_interval=args.health_check_interval,
|
|
||||||
max_failures=args.health_max_failures,
|
|
||||||
recovery_interval=args.health_recovery_interval,
|
|
||||||
)
|
|
||||||
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
|
|
||||||
|
|
||||||
summary = asyncio.run(run_benchmark(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
lb=lb,
|
|
||||||
health_checker=health_checker,
|
|
||||||
model=args.model,
|
|
||||||
concurrency=args.concurrency,
|
|
||||||
input_len=args.input_len,
|
|
||||||
output_len=args.output_len,
|
|
||||||
num_prompts=args.num_prompts,
|
|
||||||
temperature=args.temperature,
|
|
||||||
output_file=args.output_file,
|
|
||||||
))
|
|
||||||
|
|
||||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,84 +0,0 @@
|
|||||||
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
|
|
||||||
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
|
|
||||||
#
|
|
||||||
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
|
|
||||||
# Service 1: port 30005, GPUs 0,1
|
|
||||||
# Service 2: port 30006, GPUs 2,3
|
|
||||||
# Service 3: port 30007, GPUs 4,5
|
|
||||||
# Service 4: port 30008, GPUs 6,7
|
|
||||||
#
|
|
||||||
# bench_client.py distributes requests across all 4 services via round-robin
|
|
||||||
# load balancing, giving accurate cluster-wide throughput even at low
|
|
||||||
# per-service concurrency.
|
|
||||||
|
|
||||||
EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench_no_fp8"
|
|
||||||
MODEL_NAME="DeepSeek-V4-Flash"
|
|
||||||
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
||||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
||||||
|
|
||||||
# Primary port (backward compatible); multi-service ports are in PORTS.
|
|
||||||
PORT="30005"
|
|
||||||
PORTS="30005,30006,30007,30008"
|
|
||||||
|
|
||||||
BACKEND="vllm"
|
|
||||||
ENGINE="vllm"
|
|
||||||
|
|
||||||
# Native virtual environments on the host.
|
|
||||||
VENV_SERVER="/data/user1/yy/envs/vllm"
|
|
||||||
VENV_CLIENT="/data/user1/yy/envs/vllm"
|
|
||||||
|
|
||||||
# Load balancing strategy: round_robin | random | least_conn
|
|
||||||
LB_STRATEGY="round_robin"
|
|
||||||
|
|
||||||
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
|
||||||
# For multi-service we include low-concurrency scenarios to show how
|
|
||||||
# requests spread across 4 services.
|
|
||||||
SCENARIOS=(
|
|
||||||
"1 512 256 20"
|
|
||||||
"2 512 256 40"
|
|
||||||
"4 512 256 80"
|
|
||||||
"8 512 256 160"
|
|
||||||
"16 512 256 320"
|
|
||||||
"32 512 256 640"
|
|
||||||
"64 512 256 1280"
|
|
||||||
"128 512 256 2560"
|
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
|
||||||
"32 2048 512 640"
|
|
||||||
"128 2048 512 2560"
|
|
||||||
"1 4096 1024 20"
|
|
||||||
"8 4096 1024 160"
|
|
||||||
"32 4096 1024 640"
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
|
||||||
"8 8192 1024 160"
|
|
||||||
"32 8192 1024 640"
|
|
||||||
"128 8192 1024 2560"
|
|
||||||
"1 16384 1024 20"
|
|
||||||
"8 16384 1024 160"
|
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
|
||||||
"1 32768 1024 20"
|
|
||||||
"8 32768 1024 160"
|
|
||||||
"32 32768 1024 640"
|
|
||||||
"128 32768 1024 2560"
|
|
||||||
"1 65536 1024 20"
|
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
|
||||||
"128 65536 1024 2560"
|
|
||||||
"1 131072 1024 20"
|
|
||||||
"8 131072 1024 160"
|
|
||||||
"32 131072 1024 640"
|
|
||||||
"128 131072 1024 2560"
|
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
|
||||||
"32 262144 1024 640"
|
|
||||||
"128 262144 1024 2560"
|
|
||||||
"1 524288 1024 20"
|
|
||||||
"8 524288 1024 160"
|
|
||||||
"32 524288 1024 640"
|
|
||||||
"128 524288 1024 2560"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
|
||||||
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"
|
|
||||||
@ -1,206 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
|
|
||||||
|
|
||||||
Reads JSONL files produced by bench_client.py (one request per line,
|
|
||||||
last line is the summary) and generates:
|
|
||||||
- results.json (appended scenarios, following the standard schema)
|
|
||||||
- report.md (human-readable summary)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python3 parse_results.py <result_root>
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
|
|
||||||
"""Read the summary JSON (last line) and all per-request lines."""
|
|
||||||
requests: list[dict] = []
|
|
||||||
summary: dict | None = None
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
for line in f:
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
obj = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
# The summary line has "completed" and "duration" keys.
|
|
||||||
if "completed" in obj and "duration" in obj:
|
|
||||||
summary = obj
|
|
||||||
else:
|
|
||||||
requests.append(obj)
|
|
||||||
return summary, requests
|
|
||||||
|
|
||||||
|
|
||||||
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
|
|
||||||
completed = summary.get("completed", 0)
|
|
||||||
total = len(requests)
|
|
||||||
failed = total - completed if total > 0 else 0
|
|
||||||
duration_s = summary.get("duration", 0.0)
|
|
||||||
|
|
||||||
# Extract per-request metrics for percentiles.
|
|
||||||
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
|
|
||||||
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
|
|
||||||
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
|
|
||||||
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": completed,
|
|
||||||
"failed": failed,
|
|
||||||
"duration_s": duration_s,
|
|
||||||
"request_throughput": summary.get("request_throughput", 0.0),
|
|
||||||
"input_token_throughput": summary.get("input_throughput", 0.0),
|
|
||||||
"output_token_throughput": summary.get("output_throughput", 0.0),
|
|
||||||
"total_token_throughput": summary.get("total_throughput", 0.0),
|
|
||||||
"total_input_tokens": summary.get("total_input_tokens", 0),
|
|
||||||
"total_output_tokens": summary.get("total_output_tokens", 0),
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": summary.get("mean_e2e_latency_ms", 0.0),
|
|
||||||
"p50": percentile(e2es, 50),
|
|
||||||
"p90": percentile(e2es, 90),
|
|
||||||
"p95": percentile(e2es, 95),
|
|
||||||
"p99": percentile(e2es, 99),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": summary.get("mean_ttft_ms", 0.0),
|
|
||||||
"p50": percentile(ttfts, 50),
|
|
||||||
"p90": percentile(ttfts, 90),
|
|
||||||
"p95": percentile(ttfts, 95),
|
|
||||||
"p99": percentile(ttfts, 99),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": summary.get("mean_tpot_ms", 0.0),
|
|
||||||
"p50": percentile(tpots, 50),
|
|
||||||
"p90": percentile(tpots, 90),
|
|
||||||
"p95": percentile(tpots, 95),
|
|
||||||
"p99": percentile(tpots, 99),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": summary.get("mean_itl_ms", 0.0),
|
|
||||||
"p50": percentile(itls, 50),
|
|
||||||
"p90": percentile(itls, 90),
|
|
||||||
"p95": percentile(itls, 95),
|
|
||||||
"p99": percentile(itls, 99),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
|
||||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
|
||||||
|
|
||||||
|
|
||||||
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
|
|
||||||
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
|
|
||||||
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
|
|
||||||
if ttft_ok and tpot_ok:
|
|
||||||
mark = "✅"
|
|
||||||
elif ttft_ok or tpot_ok:
|
|
||||||
mark = "⚠️"
|
|
||||||
else:
|
|
||||||
mark = "❌"
|
|
||||||
return {
|
|
||||||
"ttft_p95_ok": ttft_ok,
|
|
||||||
"tpot_mean_ok": tpot_ok,
|
|
||||||
"overall": mark,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
|
||||||
with open(results_json, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
data["scenarios"].append(scenario)
|
|
||||||
with open(results_json, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
|
|
||||||
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
|
|
||||||
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
|
|
||||||
f.write(f"- **Result root**: `{result_root}`\n\n")
|
|
||||||
|
|
||||||
f.write("## Results\n\n")
|
|
||||||
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
|
||||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
|
||||||
|
|
||||||
for s in scenarios:
|
|
||||||
cfg = s["config"]
|
|
||||||
m = s["metrics"]
|
|
||||||
slo = s.get("slo_status", {}).get("overall", "")
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
|
||||||
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
|
|
||||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
|
||||||
f"{m['total_token_throughput']:.2f} | "
|
|
||||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
|
||||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
|
||||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
|
||||||
)
|
|
||||||
f.write("\n")
|
|
||||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
results_json = result_root / "results.json"
|
|
||||||
|
|
||||||
if not raw_dir.exists():
|
|
||||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
|
|
||||||
parts = jsonl_path.stem.split("_")
|
|
||||||
if len(parts) < 5:
|
|
||||||
continue
|
|
||||||
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
|
|
||||||
|
|
||||||
summary, requests = parse_jsonl(jsonl_path)
|
|
||||||
if summary is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
metrics = compute_metrics(summary, requests)
|
|
||||||
scenario = {
|
|
||||||
"name": scenario_name(concurrency, input_len, output_len),
|
|
||||||
"config": {
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"dataset": "random",
|
|
||||||
"num_prompts": metrics["success"] + metrics["failed"],
|
|
||||||
},
|
|
||||||
"metrics": metrics,
|
|
||||||
"slo_status": slo_status(metrics),
|
|
||||||
"raw_file": str(jsonl_path),
|
|
||||||
}
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
if not scenarios:
|
|
||||||
print("No benchmark outputs found to parse")
|
|
||||||
return
|
|
||||||
|
|
||||||
if results_json.exists():
|
|
||||||
for s in scenarios:
|
|
||||||
append_scenario(results_json, s)
|
|
||||||
|
|
||||||
generate_report(result_root, scenarios)
|
|
||||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
|
||||||
"run_id": "20260710-091141",
|
|
||||||
"timestamp": "2026-07-10T09:11:41+00:00",
|
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
|
||||||
"backend": "vllm",
|
|
||||||
"engine": "vllm",
|
|
||||||
"hardware": "8x NVIDIA H200 143GB",
|
|
||||||
"accelerator": "NVIDIA H200",
|
|
||||||
"chip": "nvidia_h200",
|
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
|
||||||
"git_commit": "39fb076",
|
|
||||||
"git_dirty": "dirty",
|
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20",
|
|
||||||
"4 512 256 80",
|
|
||||||
"8 512 256 160",
|
|
||||||
"16 512 256 320",
|
|
||||||
"32 512 256 640",
|
|
||||||
"64 512 256 1280",
|
|
||||||
"128 512 256 2560",
|
|
||||||
"1 2048 512 20",
|
|
||||||
"8 2048 512 160",
|
|
||||||
"32 2048 512 640",
|
|
||||||
"128 2048 512 2560",
|
|
||||||
"1 4096 1024 20",
|
|
||||||
"8 4096 1024 160",
|
|
||||||
"32 4096 1024 640",
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"scenarios": []
|
|
||||||
}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
|
||||||
"run_id": "20260710-091522",
|
|
||||||
"timestamp": "2026-07-10T09:15:22+00:00",
|
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
|
||||||
"backend": "vllm",
|
|
||||||
"engine": "vllm",
|
|
||||||
"hardware": "8x NVIDIA H200 143GB",
|
|
||||||
"accelerator": "NVIDIA H200",
|
|
||||||
"chip": "nvidia_h200",
|
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
|
||||||
"git_commit": "39fb076",
|
|
||||||
"git_dirty": "dirty",
|
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20",
|
|
||||||
"4 512 256 80",
|
|
||||||
"8 512 256 160",
|
|
||||||
"16 512 256 320",
|
|
||||||
"32 512 256 640",
|
|
||||||
"64 512 256 1280",
|
|
||||||
"128 512 256 2560",
|
|
||||||
"1 2048 512 20",
|
|
||||||
"8 2048 512 160",
|
|
||||||
"32 2048 512 640",
|
|
||||||
"128 2048 512 2560",
|
|
||||||
"1 4096 1024 20",
|
|
||||||
"8 4096 1024 160",
|
|
||||||
"32 4096 1024 640",
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"scenarios": []
|
|
||||||
}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
{
|
|
||||||
"metadata": {
|
|
||||||
"experiment": "dsv4_h200_vllm_tp2_custom_bench",
|
|
||||||
"run_id": "20260710-091700",
|
|
||||||
"timestamp": "2026-07-10T09:17:00+00:00",
|
|
||||||
"model": "/data/models/DeepSeek-V4-Flash",
|
|
||||||
"backend": "vllm",
|
|
||||||
"engine": "vllm",
|
|
||||||
"hardware": "8x NVIDIA H200 143GB",
|
|
||||||
"accelerator": "NVIDIA H200",
|
|
||||||
"chip": "nvidia_h200",
|
|
||||||
"script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh",
|
|
||||||
"env": "/data/user1/yy/envs/vllm",
|
|
||||||
"git_commit": "39fb076",
|
|
||||||
"git_dirty": "dirty",
|
|
||||||
"description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py"
|
|
||||||
},
|
|
||||||
"config": {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20",
|
|
||||||
"4 512 256 80",
|
|
||||||
"8 512 256 160",
|
|
||||||
"16 512 256 320",
|
|
||||||
"32 512 256 640",
|
|
||||||
"64 512 256 1280",
|
|
||||||
"128 512 256 2560",
|
|
||||||
"1 2048 512 20",
|
|
||||||
"8 2048 512 160",
|
|
||||||
"32 2048 512 640",
|
|
||||||
"128 2048 512 2560",
|
|
||||||
"1 4096 1024 20",
|
|
||||||
"8 4096 1024 160",
|
|
||||||
"32 4096 1024 640",
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"scenarios": []
|
|
||||||
}
|
|
||||||
@ -1,277 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200.
|
|
||||||
#
|
|
||||||
# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7).
|
|
||||||
# bench_client.py distributes requests across all 4 services via round-robin LB,
|
|
||||||
# giving accurate cluster-wide throughput even at low per-service concurrency.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# bash run_bench.sh
|
|
||||||
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
|
||||||
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
||||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
|
||||||
RAW_DIR="${RESULT_ROOT}/raw_outputs"
|
|
||||||
LOG_DIR="${RESULT_ROOT}/logs"
|
|
||||||
|
|
||||||
ensure_result_root "$RESULT_ROOT"
|
|
||||||
log_init "${LOG_DIR}/orchestrator.log"
|
|
||||||
|
|
||||||
log "experiment=${EXPERIMENT_NAME}"
|
|
||||||
log "run_id=${RUN_ID}"
|
|
||||||
log "result_root=${RESULT_ROOT}"
|
|
||||||
log "platform=${PLATFORM}"
|
|
||||||
log "chip=${CHIP}"
|
|
||||||
log "accelerator=${ACCELERATOR}"
|
|
||||||
log "engine=${ENGINE}"
|
|
||||||
log "hardware=${HARDWARE}"
|
|
||||||
log "model=${MODEL_PATH}"
|
|
||||||
log "services=${PORTS}"
|
|
||||||
log "lb_strategy=${LB_STRATEGY}"
|
|
||||||
log "server_start_script=${SERVER_START_SCRIPT}"
|
|
||||||
|
|
||||||
# Write initial metadata for this run.
|
|
||||||
METADATA_JSON="${RESULT_ROOT}/results.json"
|
|
||||||
write_metadata_json \
|
|
||||||
"$METADATA_JSON" \
|
|
||||||
"$EXPERIMENT_NAME" \
|
|
||||||
"$RUN_ID" \
|
|
||||||
"$MODEL_PATH" \
|
|
||||||
"$BACKEND" \
|
|
||||||
"$ENGINE" \
|
|
||||||
"$HARDWARE" \
|
|
||||||
"$ACCELERATOR" \
|
|
||||||
"$CHIP" \
|
|
||||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
|
||||||
"$VENV_SERVER" \
|
|
||||||
"H200 4x vLLM TP=2 multi-service (no FP8 KV cache) benchmark using bench_client.py"
|
|
||||||
|
|
||||||
# Update metadata with config.
|
|
||||||
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
|
|
||||||
import json, sys
|
|
||||||
path = sys.argv[1]
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
data["config"] = {
|
|
||||||
"tp": 2,
|
|
||||||
"num_services": 4,
|
|
||||||
"ports": "30005,30006,30007,30008",
|
|
||||||
"gpu_pairs": "0,1|2,3|4,5|6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20", "4 512 256 80", "8 512 256 160",
|
|
||||||
"16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560",
|
|
||||||
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
|
|
||||||
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
with open(path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
PY
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Server helpers (multi-service)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
is_server_healthy() {
|
|
||||||
local port="$1"
|
|
||||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
are_all_services_healthy() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if ! is_server_healthy "$p"; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_all_servers() {
|
|
||||||
log "stopping all vllm services"
|
|
||||||
# Kill by PID files first.
|
|
||||||
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
|
|
||||||
[[ -f "$pid_file" ]] || continue
|
|
||||||
local pid
|
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "killing pid=${pid}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
done
|
|
||||||
# Fallback: kill any remaining vllm processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
}
|
|
||||||
|
|
||||||
start_all_servers() {
|
|
||||||
log "starting all vllm services with ${SERVER_START_SCRIPT}"
|
|
||||||
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
|
|
||||||
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
|
|
||||||
|
|
||||||
if ! are_all_services_healthy "$PORTS"; then
|
|
||||||
log "error: not all vllm services became healthy"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
log "all vllm services are healthy"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Pre-benchmark health check with detailed reporting
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_services_detailed() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy_count=0
|
|
||||||
local unhealthy_ports=""
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
((healthy_count++))
|
|
||||||
else
|
|
||||||
unhealthy_ports="${unhealthy_ports}${p} "
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo "$healthy_count"
|
|
||||||
if [[ -n "$unhealthy_ports" ]]; then
|
|
||||||
echo "unhealthy: ${unhealthy_ports}" >&2
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_all_services() {
|
|
||||||
local ports="$1"
|
|
||||||
local max_wait="${2:-240}"
|
|
||||||
local interval="${3:-5}"
|
|
||||||
local elapsed=0
|
|
||||||
while (( elapsed < max_wait )); do
|
|
||||||
local healthy_count
|
|
||||||
healthy_count="$(check_services_detailed "$ports" | head -1)"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
if (( healthy_count == ${#PORT_LIST[@]} )); then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
|
|
||||||
sleep "$interval"
|
|
||||||
((elapsed += interval))
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Mid-benchmark health check (called between scenarios)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_and_report_services() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy=()
|
|
||||||
local unhealthy=()
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
healthy+=("$p")
|
|
||||||
else
|
|
||||||
unhealthy+=("$p")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
|
|
||||||
if [[ ${#unhealthy[@]} -gt 0 ]]; then
|
|
||||||
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
|
|
||||||
fi
|
|
||||||
return ${#unhealthy[@]}
|
|
||||||
}
|
|
||||||
|
|
||||||
on_exit() {
|
|
||||||
local code=$?
|
|
||||||
log "orchestrator exiting with code=${code}"
|
|
||||||
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
stop_all_servers
|
|
||||||
fi
|
|
||||||
exit "$code"
|
|
||||||
}
|
|
||||||
trap on_exit EXIT
|
|
||||||
|
|
||||||
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
|
|
||||||
if ! wait_for_all_services "$PORTS"; then
|
|
||||||
log "error: not all healthy services found on ports ${PORTS}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
stop_all_servers
|
|
||||||
start_all_servers
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "===== BENCHMARK START ====="
|
|
||||||
|
|
||||||
for scenario in "${SCENARIOS[@]}"; do
|
|
||||||
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
|
||||||
# Fallback for legacy 3-field scenarios.
|
|
||||||
if [[ -z "${num_prompts:-}" ]]; then
|
|
||||||
num_prompts="$NUM_PROMPTS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
|
||||||
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
|
|
||||||
|
|
||||||
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
|
|
||||||
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports "$PORTS" \
|
|
||||||
--model "$SERVED_MODEL_NAME" \
|
|
||||||
--concurrency "$concurrency" \
|
|
||||||
--input-len "$input_len" \
|
|
||||||
--output-len "$output_len" \
|
|
||||||
--num-prompts "$num_prompts" \
|
|
||||||
--lb-strategy "$LB_STRATEGY" \
|
|
||||||
--health-check-interval 5 \
|
|
||||||
--health-max-failures 2 \
|
|
||||||
--health-recovery-interval 10 \
|
|
||||||
--request-max-retries 2 \
|
|
||||||
--output-file "$output_file" \
|
|
||||||
> "$detail_log" 2>&1 || {
|
|
||||||
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
|
||||||
# After failure, check service health before continuing.
|
|
||||||
check_and_report_services "$PORTS"
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log "finished scenario: output=${output_file}"
|
|
||||||
|
|
||||||
# Between scenarios, report service health so we can spot degradation early.
|
|
||||||
check_and_report_services "$PORTS" || true
|
|
||||||
done
|
|
||||||
|
|
||||||
log "===== BENCHMARK DONE ====="
|
|
||||||
|
|
||||||
log "parsing results"
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
|
|
||||||
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
|
|
||||||
}
|
|
||||||
|
|
||||||
log "all results saved to ${RESULT_ROOT}"
|
|
||||||
@ -1,109 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Start 4 independent vLLM TP=2 services on 8x H200.
|
|
||||||
# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7).
|
|
||||||
# This maximizes GPU utilization for low-concurrency workloads.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
VENV="${VENV_SERVER}"
|
|
||||||
export PATH="$VENV/bin:$PATH"
|
|
||||||
VLLM="$VENV/bin/vllm"
|
|
||||||
|
|
||||||
TP=2
|
|
||||||
LOG_DIR="${SCRIPT_DIR}/logs"
|
|
||||||
mkdir -p "$LOG_DIR"
|
|
||||||
|
|
||||||
# 4 services, each on 2 GPUs.
|
|
||||||
# Port and GPU pair mapping.
|
|
||||||
SERVICES=(
|
|
||||||
"30005:0,1"
|
|
||||||
"30006:2,3"
|
|
||||||
"30007:4,5"
|
|
||||||
"30008:6,7"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Kill any existing vLLM processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
# Remove old PID files.
|
|
||||||
rm -f "${SCRIPT_DIR}"/*.pid
|
|
||||||
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
GPUS="${svc##*:}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
|
|
||||||
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
|
|
||||||
echo "Log: ${LOG}"
|
|
||||||
|
|
||||||
CUDA_VISIBLE_DEVICES="${GPUS}" \
|
|
||||||
nohup "$VLLM" serve "$MODEL_PATH" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--tensor-parallel-size "$TP" \
|
|
||||||
--block-size 256 \
|
|
||||||
--served-model-name "$SERVED_MODEL_NAME" \
|
|
||||||
--port "$PORT" \
|
|
||||||
> "$LOG" 2>&1 &
|
|
||||||
PID=$!
|
|
||||||
echo $PID > "$PID_FILE"
|
|
||||||
echo "PID: $PID"
|
|
||||||
done
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Waiting for all 4 services to become healthy..."
|
|
||||||
|
|
||||||
MAX_WAIT=240
|
|
||||||
all_healthy=0
|
|
||||||
for i in $(seq 1 $MAX_WAIT); do
|
|
||||||
all_healthy=1
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
|
|
||||||
all_healthy=0
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$all_healthy" -eq 1 ]]; then
|
|
||||||
echo "All 4 services are healthy!"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
echo " - http://127.0.0.1:${PORT}"
|
|
||||||
done
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if any process died early.
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
if [[ -f "$PID_FILE" ]]; then
|
|
||||||
PID="$(cat "$PID_FILE")"
|
|
||||||
if ! kill -0 "$PID" 2>/dev/null; then
|
|
||||||
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
tail -200 $LOG 2>/dev/null || true
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if (( i % 10 == 0 )); then
|
|
||||||
echo "Waiting... (${i}/${MAX_WAIT})"
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
echo "--- Last 50 lines of port ${PORT} ---"
|
|
||||||
tail -50 $LOG 2>/dev/null || true
|
|
||||||
done
|
|
||||||
exit 1
|
|
||||||
@ -228,16 +228,22 @@ class LoadBalancer:
|
|||||||
# Token helpers
|
# Token helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
import uuid
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
def make_prompt(token_len: int) -> str:
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
prefix = str(uuid.uuid4())
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
words = [
|
||||||
needed = max(token_len, 1)
|
"the", "quick", "brown", "fox",
|
||||||
tokens: list[str] = []
|
"jumps", "over", "lazy", "dog",
|
||||||
while len(tokens) < needed:
|
"benchmark", "performance", "latency"
|
||||||
|
]
|
||||||
|
|
||||||
|
tokens = [prefix]
|
||||||
|
|
||||||
|
while len(tokens) < token_len:
|
||||||
tokens.extend(words)
|
tokens.extend(words)
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
return " ".join(tokens[:token_len])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@ -440,7 +446,36 @@ async def run_benchmark(
|
|||||||
return await send_request_with_retry(
|
return await send_request_with_retry(
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
session, host, lb, model, idx, prompt, output_len, temperature
|
||||||
)
|
)
|
||||||
|
print("Starting warmup...")
|
||||||
|
warmup_per_service = 10
|
||||||
|
warmup_requests = warmup_per_service * len(ports)
|
||||||
|
|
||||||
|
warmup_prompts = [
|
||||||
|
make_prompt(input_len)
|
||||||
|
for _ in range(warmup_requests)
|
||||||
|
]
|
||||||
|
|
||||||
|
warmup_tasks = [
|
||||||
|
send_request_with_retry(
|
||||||
|
session=session,
|
||||||
|
host=host,
|
||||||
|
lb=lb,
|
||||||
|
model=model,
|
||||||
|
request_id=-1,
|
||||||
|
prompt=p,
|
||||||
|
max_tokens=output_len,
|
||||||
|
temperature=temperature,
|
||||||
|
)
|
||||||
|
for p in warmup_prompts
|
||||||
|
]
|
||||||
|
|
||||||
|
await asyncio.gather(*warmup_tasks)
|
||||||
|
|
||||||
|
print("Warmup finished.")
|
||||||
|
|
||||||
|
# 给 CUDA / Scheduler 一点时间稳定
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
await health_checker.stop()
|
await health_checker.stop()
|
||||||
|
|||||||
@ -34,50 +34,146 @@ LB_STRATEGY="round_robin"
|
|||||||
# For multi-service we include low-concurrency scenarios to show how
|
# For multi-service we include low-concurrency scenarios to show how
|
||||||
# requests spread across 2 services.
|
# requests spread across 2 services.
|
||||||
SCENARIOS=(
|
SCENARIOS=(
|
||||||
"1 512 256 20"
|
"1 512 256 20"
|
||||||
"2 512 256 40"
|
"2 512 256 40"
|
||||||
"4 512 256 80"
|
"4 512 256 80"
|
||||||
"8 512 256 160"
|
"8 512 256 160"
|
||||||
"16 512 256 320"
|
"16 512 256 320"
|
||||||
"32 512 256 640"
|
"32 512 256 640"
|
||||||
"64 512 256 1280"
|
"64 512 256 1280"
|
||||||
"128 512 256 2560"
|
"128 512 256 2560"
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
"1 1024 128 20"
|
||||||
"32 2048 512 640"
|
"1 1024 256 20"
|
||||||
"128 2048 512 2560"
|
"1 1024 512 20"
|
||||||
"1 4096 1024 20"
|
"1 1024 1024 20"
|
||||||
"8 4096 1024 160"
|
"1 1024 2048 20"
|
||||||
"32 4096 1024 640"
|
"1 1024 4096 20"
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
"8 1024 128 160"
|
||||||
"8 8192 1024 160"
|
"8 1024 256 160"
|
||||||
"32 8192 1024 640"
|
"8 1024 512 160"
|
||||||
"128 8192 1024 2560"
|
"8 1024 1024 160"
|
||||||
"1 16384 1024 20"
|
"8 1024 2048 160"
|
||||||
"8 16384 1024 160"
|
"8 1024 4096 160"
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
"32 1024 128 640"
|
||||||
"1 32768 1024 20"
|
"32 1024 256 640"
|
||||||
"8 32768 1024 160"
|
"32 1024 512 640"
|
||||||
"32 32768 1024 640"
|
"32 1024 1024 640"
|
||||||
"128 32768 1024 2560"
|
"32 1024 2048 640"
|
||||||
"1 65536 1024 20"
|
"32 1024 4096 640"
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
"128 1024 128 2560"
|
||||||
"128 65536 1024 2560"
|
"128 1024 256 2560"
|
||||||
"1 131072 1024 20"
|
"128 1024 512 2560"
|
||||||
"8 131072 1024 160"
|
"128 1024 1024 2560"
|
||||||
"32 131072 1024 640"
|
"128 1024 2048 2560"
|
||||||
"128 131072 1024 2560"
|
"128 1024 4096 2560"
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
"1 2048 128 20"
|
||||||
"32 262144 1024 640"
|
"1 2048 256 20"
|
||||||
"128 262144 1024 2560"
|
"1 2048 512 20"
|
||||||
"1 524288 1024 20"
|
"1 2048 1024 20"
|
||||||
"8 524288 1024 160"
|
"1 2048 2048 20"
|
||||||
"32 524288 1024 640"
|
"1 2048 4096 20"
|
||||||
"128 524288 1024 2560"
|
|
||||||
|
"8 2048 128 160"
|
||||||
|
"8 2048 256 160"
|
||||||
|
"8 2048 512 160"
|
||||||
|
"8 2048 1024 160"
|
||||||
|
"8 2048 2048 160"
|
||||||
|
"8 2048 4096 160"
|
||||||
|
|
||||||
|
"32 2048 128 640"
|
||||||
|
"32 2048 256 640"
|
||||||
|
"32 2048 512 640"
|
||||||
|
"32 2048 1024 640"
|
||||||
|
"32 2048 2048 640"
|
||||||
|
"32 2048 4096 640"
|
||||||
|
|
||||||
|
"128 2048 128 2560"
|
||||||
|
"128 2048 256 2560"
|
||||||
|
"128 2048 512 2560"
|
||||||
|
"128 2048 1024 2560"
|
||||||
|
"128 2048 2048 2560"
|
||||||
|
"128 2048 4096 2560"
|
||||||
|
|
||||||
|
"1 4096 128 20"
|
||||||
|
"1 4096 256 20"
|
||||||
|
"1 4096 512 20"
|
||||||
|
"1 4096 1024 20"
|
||||||
|
"1 4096 2048 20"
|
||||||
|
"1 4096 4096 20"
|
||||||
|
|
||||||
|
"8 4096 128 160"
|
||||||
|
"8 4096 256 160"
|
||||||
|
"8 4096 512 160"
|
||||||
|
"8 4096 1024 160"
|
||||||
|
"8 4096 2048 160"
|
||||||
|
"8 4096 4096 160"
|
||||||
|
|
||||||
|
"32 4096 128 640"
|
||||||
|
"32 4096 256 640"
|
||||||
|
"32 4096 512 640"
|
||||||
|
"32 4096 1024 640"
|
||||||
|
"32 4096 2048 640"
|
||||||
|
"32 4096 4096 640"
|
||||||
|
|
||||||
|
"128 4096 128 2560"
|
||||||
|
"128 4096 256 2560"
|
||||||
|
"128 4096 512 2560"
|
||||||
|
"128 4096 1024 2560"
|
||||||
|
"128 4096 2048 2560"
|
||||||
|
"128 4096 4096 2560"
|
||||||
|
|
||||||
|
"1 16384 128 20"
|
||||||
|
"1 16384 256 20"
|
||||||
|
"1 16384 512 20"
|
||||||
|
"1 16384 1024 20"
|
||||||
|
"1 16384 2048 20"
|
||||||
|
|
||||||
|
"8 16384 128 160"
|
||||||
|
"8 16384 256 160"
|
||||||
|
"8 16384 512 160"
|
||||||
|
"8 16384 1024 160"
|
||||||
|
"8 16384 2048 160"
|
||||||
|
|
||||||
|
"32 16384 128 640"
|
||||||
|
"32 16384 256 640"
|
||||||
|
"32 16384 512 640"
|
||||||
|
"32 16384 1024 640"
|
||||||
|
"32 16384 2048 640"
|
||||||
|
|
||||||
|
"128 16384 128 2560"
|
||||||
|
"128 16384 256 2560"
|
||||||
|
"128 16384 512 2560"
|
||||||
|
"128 16384 1024 2560"
|
||||||
|
"128 16384 2048 2560"
|
||||||
|
|
||||||
|
|
||||||
|
"1 65536 128 20"
|
||||||
|
"1 65536 256 20"
|
||||||
|
"1 65536 512 20"
|
||||||
|
"1 65536 1024 20"
|
||||||
|
|
||||||
|
"8 65536 128 160"
|
||||||
|
"8 65536 256 160"
|
||||||
|
"8 65536 512 160"
|
||||||
|
"8 65536 1024 160"
|
||||||
|
"32 65536 1024 160"
|
||||||
|
|
||||||
|
"1 131072 128 20"
|
||||||
|
"1 131072 256 20"
|
||||||
|
"1 131072 512 20"
|
||||||
|
|
||||||
|
"8 131072 512 160"
|
||||||
|
|
||||||
|
"1 262144 256 20"
|
||||||
|
|
||||||
|
"8 262144 128 160"
|
||||||
|
|
||||||
|
"1 524288 128 20"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
# Server start script bundled with this experiment.
|
||||||
|
|||||||
@ -0,0 +1,129 @@
|
|||||||
|
# H200 vLLM TP=2 Custom Benchmark Report
|
||||||
|
|
||||||
|
- **Client**: `bench_client.py` (async OpenAI API, per-request timing)
|
||||||
|
- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)
|
||||||
|
- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908`
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||||
|
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| c128_i1024_o1024 | 128 | 1024 | 1024 | 172.39 | 2560 | 0 | 14.85 | 15206.17 | 3100.60 | 18306.76 | 267.16 | 478.80 | 2297.70 | 38.53 | 66.16 | 70.07 | 8265.00 | 13967.23 | 17536.46 | ✅ |
|
||||||
|
| c128_i1024_o128 | 128 | 1024 | 128 | 98.84 | 2560 | 0 | 25.90 | 26522.07 | 3314.91 | 29836.98 | 900.39 | 1590.87 | 2307.93 | 30.16 | 35.98 | 37.09 | 4730.89 | 5530.64 | 6122.15 | ✅ |
|
||||||
|
| c128_i1024_o2048 | 128 | 1024 | 2048 | 161.76 | 2560 | 0 | 15.83 | 16205.28 | 3297.74 | 19503.02 | 256.61 | 340.87 | 2315.99 | 36.07 | 39.27 | 40.30 | 7732.01 | 10616.59 | 11870.42 | ✅ |
|
||||||
|
| c128_i1024_o256 | 128 | 1024 | 256 | 161.16 | 2560 | 0 | 15.89 | 16266.52 | 3250.72 | 19517.24 | 259.69 | 356.25 | 2333.30 | 36.68 | 40.81 | 51.99 | 7727.17 | 10133.95 | 11225.21 | ✅ |
|
||||||
|
| c128_i1024_o4096 | 128 | 1024 | 4096 | 163.16 | 2560 | 0 | 15.69 | 16066.27 | 3281.03 | 19347.31 | 257.28 | 422.58 | 2303.92 | 36.09 | 39.21 | 40.35 | 7768.07 | 10730.88 | 12234.15 | ✅ |
|
||||||
|
| c128_i1024_o512 | 128 | 1024 | 512 | 161.49 | 2560 | 0 | 15.85 | 16232.48 | 3275.92 | 19508.40 | 255.16 | 334.67 | 2297.21 | 36.27 | 39.39 | 40.57 | 7716.22 | 10581.99 | 11832.60 | ✅ |
|
||||||
|
| c128_i16384_o1024 | 128 | 16384 | 1024 | 919.25 | 2560 | 0 | 2.78 | 45627.36 | 697.33 | 46324.69 | 2625.93 | 3630.11 | 33634.45 | 171.30 | 196.69 | 207.71 | 45001.23 | 90438.26 | 123707.69 | ❌ |
|
||||||
|
| c128_i16384_o128 | 128 | 16384 | 128 | 859.37 | 2560 | 0 | 2.98 | 48806.72 | 380.95 | 49187.68 | 2833.19 | 3131.25 | 33643.84 | 311.88 | 328.04 | 328.10 | 42403.99 | 44239.06 | 73426.74 | ❌ |
|
||||||
|
| c128_i16384_o2048 | 128 | 16384 | 2048 | 918.61 | 2560 | 0 | 2.79 | 45659.28 | 701.99 | 46361.27 | 2601.36 | 3607.43 | 33624.66 | 170.03 | 195.00 | 208.79 | 45030.71 | 92401.43 | 124284.89 | ❌ |
|
||||||
|
| c128_i16384_o256 | 128 | 16384 | 256 | 897.62 | 2560 | 0 | 2.85 | 46726.87 | 602.42 | 47329.29 | 2804.71 | 4596.59 | 33617.37 | 197.06 | 224.66 | 236.73 | 44219.84 | 57755.16 | 76587.71 | ❌ |
|
||||||
|
| c128_i16384_o512 | 128 | 16384 | 512 | 915.13 | 2560 | 0 | 2.80 | 45833.00 | 684.56 | 46517.56 | 2646.03 | 3737.57 | 33718.32 | 174.28 | 203.88 | 224.11 | 44914.54 | 88267.88 | 100313.27 | ❌ |
|
||||||
|
| c128_i2048_o1024 | 128 | 2048 | 1024 | 204.30 | 2560 | 0 | 12.53 | 25663.20 | 2671.39 | 28334.59 | 372.84 | 493.00 | 4208.03 | 44.24 | 48.11 | 50.53 | 9762.62 | 15935.84 | 21541.33 | ✅ |
|
||||||
|
| c128_i2048_o128 | 128 | 2048 | 128 | 153.29 | 2560 | 0 | 16.70 | 34202.45 | 2125.54 | 36327.99 | 791.19 | 1283.06 | 4199.42 | 52.51 | 58.15 | 59.97 | 7426.54 | 8226.85 | 10576.91 | ⚠️ |
|
||||||
|
| c128_i2048_o2048 | 128 | 2048 | 2048 | 208.80 | 2560 | 0 | 12.26 | 25109.31 | 2631.24 | 27740.55 | 365.94 | 530.02 | 4211.56 | 44.40 | 48.85 | 54.00 | 9842.38 | 16285.06 | 20408.42 | ✅ |
|
||||||
|
| c128_i2048_o256 | 128 | 2048 | 256 | 196.63 | 2560 | 0 | 13.02 | 26663.63 | 2605.54 | 29269.17 | 379.32 | 522.39 | 4205.72 | 45.73 | 50.01 | 51.66 | 9476.10 | 12538.01 | 13720.71 | ✅ |
|
||||||
|
| c128_i2048_o4096 | 128 | 2048 | 4096 | 204.98 | 2560 | 0 | 12.49 | 25578.03 | 2696.32 | 28274.35 | 369.65 | 497.47 | 4203.66 | 44.06 | 47.90 | 49.49 | 9834.14 | 15705.61 | 22444.28 | ✅ |
|
||||||
|
| c128_i2048_o512 | 128 | 2048 | 512 | 205.58 | 2560 | 0 | 12.45 | 25502.28 | 2682.24 | 28184.52 | 364.96 | 479.81 | 4198.31 | 44.32 | 48.38 | 49.46 | 9849.41 | 16273.27 | 21537.21 | ✅ |
|
||||||
|
| c128_i4096_o1024 | 128 | 4096 | 1024 | 299.56 | 2560 | 0 | 8.55 | 35004.04 | 1823.49 | 36827.53 | 670.94 | 898.57 | 8472.05 | 64.95 | 71.72 | 78.31 | 14410.80 | 23823.73 | 34763.30 | ⚠️ |
|
||||||
|
| c128_i4096_o128 | 128 | 4096 | 128 | 252.68 | 2560 | 0 | 10.13 | 41497.56 | 1283.45 | 42781.02 | 1110.40 | 1837.39 | 8464.33 | 89.25 | 99.28 | 101.99 | 12341.74 | 13909.51 | 19709.73 | ⚠️ |
|
||||||
|
| c128_i4096_o2048 | 128 | 4096 | 2048 | 299.51 | 2560 | 0 | 8.55 | 35009.54 | 1841.89 | 36851.42 | 673.64 | 903.51 | 8456.42 | 64.31 | 71.20 | 77.98 | 14467.32 | 24917.88 | 37076.16 | ⚠️ |
|
||||||
|
| c128_i4096_o256 | 128 | 4096 | 256 | 288.98 | 2560 | 0 | 8.86 | 36284.85 | 1743.96 | 38028.81 | 688.49 | 1029.32 | 8474.33 | 68.27 | 74.67 | 80.07 | 14052.64 | 18727.14 | 21844.01 | ⚠️ |
|
||||||
|
| c128_i4096_o4096 | 128 | 4096 | 4096 | 297.69 | 2560 | 0 | 8.60 | 35223.60 | 1820.84 | 37044.43 | 679.88 | 979.68 | 8471.74 | 65.18 | 72.12 | 79.04 | 14368.53 | 24055.68 | 33864.98 | ⚠️ |
|
||||||
|
| c128_i4096_o512 | 128 | 4096 | 512 | 296.90 | 2560 | 0 | 8.62 | 35317.89 | 1815.79 | 37133.68 | 666.19 | 888.77 | 8469.32 | 65.45 | 72.23 | 78.81 | 14377.25 | 24981.06 | 33674.28 | ⚠️ |
|
||||||
|
| c128_i512_o256 | 128 | 512 | 256 | 155.29 | 2560 | 0 | 16.49 | 8440.72 | 3401.04 | 11841.77 | 216.12 | 439.65 | 1351.98 | 35.17 | 38.71 | 39.83 | 7433.74 | 9800.36 | 10039.98 | ✅ |
|
||||||
|
| c16_i512_o256 | 16 | 512 | 256 | 60.82 | 320 | 0 | 5.26 | 2693.64 | 1089.67 | 3783.32 | 145.56 | 311.20 | 388.94 | 12.62 | 17.00 | 17.83 | 2748.60 | 4105.76 | 4640.23 | ✅ |
|
||||||
|
| c1_i1024_o1024 | 1 | 1024 | 1024 | 52.34 | 20 | 0 | 0.38 | 391.29 | 85.14 | 476.42 | 115.81 | 118.26 | 140.50 | 6.99 | 7.00 | 7.00 | 1666.63 | 2301.13 | 2358.69 | ✅ |
|
||||||
|
| c1_i1024_o128 | 1 | 1024 | 128 | 32.71 | 20 | 0 | 0.61 | 626.06 | 78.07 | 704.13 | 115.30 | 122.94 | 128.47 | 6.96 | 6.97 | 7.01 | 997.08 | 1004.84 | 1011.97 | ✅ |
|
||||||
|
| c1_i1024_o2048 | 1 | 1024 | 2048 | 48.50 | 20 | 0 | 0.41 | 422.30 | 83.02 | 505.31 | 115.62 | 119.50 | 126.29 | 6.99 | 7.00 | 7.01 | 1516.37 | 1700.77 | 1778.35 | ✅ |
|
||||||
|
| c1_i1024_o256 | 1 | 1024 | 256 | 48.48 | 20 | 0 | 0.41 | 422.40 | 81.26 | 503.66 | 114.81 | 120.58 | 125.23 | 6.99 | 7.01 | 7.01 | 1485.39 | 1837.55 | 1880.36 | ✅ |
|
||||||
|
| c1_i1024_o4096 | 1 | 1024 | 4096 | 51.98 | 20 | 0 | 0.38 | 393.97 | 82.72 | 476.69 | 116.51 | 126.81 | 137.83 | 6.99 | 7.00 | 7.01 | 1612.52 | 2093.77 | 2163.97 | ✅ |
|
||||||
|
| c1_i1024_o512 | 1 | 1024 | 512 | 49.68 | 20 | 0 | 0.40 | 412.23 | 78.90 | 491.13 | 132.68 | 174.48 | 175.39 | 7.00 | 7.02 | 7.02 | 1497.09 | 2052.55 | 2150.57 | ✅ |
|
||||||
|
| c1_i131072_o128 | 1 | 131072 | 128 | 257.45 | 20 | 0 | 0.08 | 10182.44 | 8.90 | 10191.34 | 6471.48 | 6574.88 | 6590.68 | 7.21 | 7.25 | 7.32 | 7290.28 | 7408.33 | 7425.83 | ⚠️ |
|
||||||
|
| c1_i131072_o256 | 1 | 131072 | 256 | 259.43 | 20 | 0 | 0.08 | 10104.51 | 13.11 | 10117.61 | 6523.80 | 6605.21 | 6683.36 | 7.25 | 7.28 | 7.30 | 7749.51 | 8208.24 | 8219.55 | ⚠️ |
|
||||||
|
| c1_i131072_o512 | 1 | 131072 | 512 | 281.03 | 20 | 0 | 0.07 | 9328.01 | 15.40 | 9343.41 | 6520.17 | 6595.29 | 6639.39 | 7.26 | 7.27 | 7.29 | 8083.13 | 9930.97 | 10147.71 | ⚠️ |
|
||||||
|
| c1_i16384_o1024 | 1 | 16384 | 1024 | 85.55 | 20 | 0 | 0.23 | 3830.19 | 66.12 | 3896.31 | 761.59 | 767.17 | 767.57 | 7.09 | 7.11 | 7.11 | 2761.83 | 4275.72 | 5529.53 | ✅ |
|
||||||
|
| c1_i16384_o128 | 1 | 16384 | 128 | 54.06 | 20 | 0 | 0.37 | 6061.09 | 46.98 | 6108.07 | 758.98 | 762.04 | 764.60 | 7.03 | 7.04 | 7.04 | 1645.31 | 1649.67 | 1651.54 | ✅ |
|
||||||
|
| c1_i16384_o2048 | 1 | 16384 | 2048 | 90.20 | 20 | 0 | 0.22 | 3632.96 | 62.71 | 3695.67 | 762.25 | 769.13 | 769.49 | 7.10 | 7.11 | 7.11 | 2762.54 | 3858.26 | 4063.08 | ✅ |
|
||||||
|
| c1_i16384_o256 | 1 | 16384 | 256 | 73.13 | 20 | 0 | 0.27 | 4480.86 | 56.19 | 4537.04 | 757.24 | 769.18 | 772.94 | 7.09 | 7.11 | 7.11 | 2205.64 | 2558.68 | 2568.89 | ✅ |
|
||||||
|
| c1_i16384_o512 | 1 | 16384 | 512 | 75.43 | 20 | 0 | 0.27 | 4344.23 | 53.96 | 4398.19 | 761.98 | 768.30 | 778.23 | 7.10 | 7.11 | 7.11 | 2198.96 | 2832.27 | 2889.46 | ✅ |
|
||||||
|
| c1_i2048_o1024 | 1 | 2048 | 1024 | 52.15 | 20 | 0 | 0.38 | 785.38 | 81.93 | 867.31 | 135.15 | 140.23 | 150.82 | 7.07 | 7.09 | 7.10 | 1639.00 | 2263.67 | 3129.84 | ✅ |
|
||||||
|
| c1_i2048_o128 | 1 | 2048 | 128 | 33.24 | 20 | 0 | 0.60 | 1232.22 | 74.97 | 1307.19 | 136.94 | 159.17 | 181.96 | 7.04 | 7.10 | 7.10 | 1006.62 | 1053.40 | 1075.84 | ✅ |
|
||||||
|
| c1_i2048_o2048 | 1 | 2048 | 2048 | 51.48 | 20 | 0 | 0.39 | 795.71 | 84.60 | 880.31 | 134.55 | 137.97 | 148.69 | 7.07 | 7.09 | 7.10 | 1667.40 | 3179.96 | 3366.92 | ✅ |
|
||||||
|
| c1_i2048_o256 | 1 | 2048 | 256 | 48.45 | 20 | 0 | 0.41 | 845.46 | 79.01 | 924.47 | 133.36 | 134.84 | 138.71 | 7.07 | 7.08 | 7.09 | 1479.65 | 1930.94 | 1931.75 | ✅ |
|
||||||
|
| c1_i2048_o4096 | 1 | 2048 | 4096 | 52.65 | 20 | 0 | 0.38 | 777.92 | 77.54 | 855.46 | 134.26 | 137.78 | 149.15 | 7.08 | 7.09 | 7.10 | 1571.48 | 2033.68 | 2095.42 | ✅ |
|
||||||
|
| c1_i2048_o512 | 1 | 2048 | 512 | 50.39 | 20 | 0 | 0.40 | 812.88 | 79.10 | 891.98 | 133.86 | 135.40 | 147.37 | 7.08 | 7.10 | 7.10 | 1537.04 | 2769.40 | 2894.64 | ✅ |
|
||||||
|
| c1_i262144_o256 | 1 | 262144 | 256 | 556.90 | 20 | 0 | 0.04 | 9414.44 | 6.27 | 9420.71 | 15234.61 | 15316.53 | 15320.27 | 7.47 | 7.55 | 7.57 | 16530.72 | 17025.05 | 17032.91 | ⚠️ |
|
||||||
|
| c1_i4096_o1024 | 1 | 4096 | 1024 | 50.77 | 20 | 0 | 0.39 | 1613.53 | 78.27 | 1691.81 | 223.98 | 228.84 | 229.98 | 7.08 | 7.09 | 7.09 | 1622.74 | 2090.63 | 2641.63 | ✅ |
|
||||||
|
| c1_i4096_o128 | 1 | 4096 | 128 | 36.18 | 20 | 0 | 0.55 | 2264.03 | 70.17 | 2334.20 | 223.37 | 225.71 | 227.21 | 7.03 | 7.04 | 7.07 | 1108.62 | 1117.89 | 1119.93 | ✅ |
|
||||||
|
| c1_i4096_o2048 | 1 | 4096 | 2048 | 54.09 | 20 | 0 | 0.37 | 1514.64 | 78.34 | 1592.98 | 223.30 | 226.85 | 227.97 | 7.08 | 7.09 | 7.09 | 1715.15 | 2736.41 | 2965.24 | ✅ |
|
||||||
|
| c1_i4096_o256 | 1 | 4096 | 256 | 53.93 | 20 | 0 | 0.37 | 1519.00 | 77.17 | 1596.18 | 224.11 | 227.58 | 229.35 | 7.07 | 7.09 | 7.09 | 1687.85 | 2022.93 | 2023.17 | ✅ |
|
||||||
|
| c1_i4096_o4096 | 1 | 4096 | 4096 | 50.60 | 20 | 0 | 0.40 | 1618.96 | 72.98 | 1691.94 | 223.43 | 228.43 | 229.73 | 7.08 | 7.09 | 7.09 | 1522.86 | 2146.74 | 2282.49 | ✅ |
|
||||||
|
| c1_i4096_o512 | 1 | 4096 | 512 | 59.75 | 20 | 0 | 0.33 | 1371.10 | 81.14 | 1452.25 | 224.35 | 229.07 | 230.84 | 7.07 | 7.08 | 7.08 | 1931.53 | 2977.28 | 3648.36 | ✅ |
|
||||||
|
| c1_i512_o256 | 1 | 512 | 256 | 53.26 | 20 | 0 | 0.38 | 192.26 | 80.81 | 273.07 | 167.28 | 203.13 | 205.33 | 6.99 | 7.02 | 7.03 | 1663.97 | 1951.39 | 1953.26 | ✅ |
|
||||||
|
| c1_i524288_o128 | 1 | 524288 | 128 | 1513.75 | 20 | 0 | 0.01 | 6927.01 | 1.20 | 6928.22 | 40602.84 | 41121.70 | 41219.49 | 7.78 | 7.92 | 7.93 | 41303.57 | 41857.14 | 41958.45 | ⚠️ |
|
||||||
|
| c1_i65536_o1024 | 1 | 65536 | 1024 | 152.81 | 20 | 0 | 0.13 | 8577.64 | 26.28 | 8603.92 | 2984.49 | 2994.56 | 3006.30 | 7.18 | 7.18 | 7.19 | 4418.27 | 5282.47 | 5593.15 | ✅ |
|
||||||
|
| c1_i65536_o128 | 1 | 65536 | 128 | 134.86 | 20 | 0 | 0.15 | 9718.82 | 17.89 | 9736.71 | 2996.65 | 3058.30 | 3078.26 | 7.11 | 7.16 | 7.16 | 3847.83 | 3911.31 | 3932.26 | ⚠️ |
|
||||||
|
| c1_i65536_o256 | 1 | 65536 | 256 | 153.56 | 20 | 0 | 0.13 | 8535.68 | 25.96 | 8561.64 | 2986.14 | 2994.62 | 3008.90 | 7.16 | 7.18 | 7.21 | 4407.24 | 4762.51 | 4778.90 | ✅ |
|
||||||
|
| c1_i65536_o512 | 1 | 65536 | 512 | 157.16 | 20 | 0 | 0.13 | 8340.12 | 27.70 | 8367.82 | 2986.06 | 2996.34 | 3023.15 | 7.18 | 7.19 | 7.20 | 4541.31 | 6107.66 | 6492.71 | ✅ |
|
||||||
|
| c2_i512_o256 | 2 | 512 | 256 | 44.94 | 40 | 0 | 0.89 | 455.73 | 188.32 | 644.06 | 123.41 | 177.91 | 187.26 | 7.06 | 7.48 | 7.68 | 1611.58 | 2020.21 | 2050.24 | ✅ |
|
||||||
|
| c32_i1024_o1024 | 32 | 1024 | 1024 | 78.93 | 640 | 0 | 8.11 | 8303.55 | 1684.20 | 9987.75 | 154.93 | 217.27 | 815.30 | 16.71 | 18.61 | 18.94 | 3611.27 | 4919.69 | 5510.54 | ✅ |
|
||||||
|
| c32_i1024_o128 | 32 | 1024 | 128 | 46.47 | 640 | 0 | 13.77 | 14103.72 | 1762.90 | 15866.62 | 408.25 | 714.24 | 757.98 | 13.41 | 16.99 | 17.83 | 2110.85 | 2431.17 | 2650.42 | ✅ |
|
||||||
|
| c32_i1024_o2048 | 32 | 1024 | 2048 | 81.00 | 640 | 0 | 7.90 | 8090.92 | 1622.21 | 9713.13 | 162.95 | 334.37 | 753.35 | 17.45 | 23.66 | 25.29 | 3730.41 | 5542.76 | 6244.95 | ✅ |
|
||||||
|
| c32_i1024_o256 | 32 | 1024 | 256 | 76.79 | 640 | 0 | 8.33 | 8534.95 | 1672.11 | 10207.06 | 154.72 | 212.02 | 753.62 | 16.97 | 19.11 | 20.00 | 3544.98 | 4746.50 | 4887.24 | ✅ |
|
||||||
|
| c32_i1024_o4096 | 32 | 1024 | 4096 | 77.85 | 640 | 0 | 8.22 | 8418.72 | 1664.78 | 10083.50 | 154.28 | 214.19 | 756.53 | 16.85 | 18.82 | 19.49 | 3549.62 | 4954.17 | 5479.06 | ✅ |
|
||||||
|
| c32_i1024_o512 | 32 | 1024 | 512 | 77.73 | 640 | 0 | 8.23 | 8431.17 | 1700.51 | 10131.68 | 155.17 | 216.90 | 751.85 | 16.62 | 18.26 | 18.67 | 3574.12 | 4831.87 | 5478.82 | ✅ |
|
||||||
|
| c32_i16384_o1024 | 32 | 16384 | 1024 | 277.96 | 640 | 0 | 2.30 | 37723.90 | 563.76 | 38287.65 | 1266.27 | 1984.25 | 8777.02 | 48.75 | 60.64 | 65.85 | 13073.64 | 26643.75 | 35597.74 | ✅ |
|
||||||
|
| c32_i16384_o128 | 32 | 16384 | 128 | 240.94 | 640 | 0 | 2.66 | 43520.72 | 338.90 | 43859.62 | 3738.21 | 4848.25 | 8794.30 | 61.61 | 81.93 | 82.76 | 11539.01 | 13150.69 | 16925.01 | ❌ |
|
||||||
|
| c32_i16384_o2048 | 32 | 16384 | 2048 | 277.48 | 640 | 0 | 2.31 | 37789.10 | 576.94 | 38366.04 | 1268.99 | 2250.82 | 8796.37 | 47.57 | 58.28 | 64.83 | 13129.88 | 24681.93 | 38211.25 | ✅ |
|
||||||
|
| c32_i16384_o256 | 32 | 16384 | 256 | 264.70 | 640 | 0 | 2.42 | 39614.37 | 508.66 | 40123.04 | 1298.03 | 1994.66 | 8761.72 | 54.06 | 64.54 | 70.67 | 12625.00 | 16418.68 | 19908.81 | ⚠️ |
|
||||||
|
| c32_i16384_o512 | 32 | 16384 | 512 | 275.93 | 640 | 0 | 2.32 | 38001.51 | 561.14 | 38562.65 | 1268.67 | 1990.86 | 8787.03 | 48.89 | 60.15 | 66.01 | 13034.17 | 25370.34 | 27540.55 | ✅ |
|
||||||
|
| c32_i2048_o1024 | 32 | 2048 | 1024 | 87.95 | 640 | 0 | 7.28 | 14903.68 | 1574.85 | 16478.54 | 200.89 | 270.09 | 1352.10 | 17.77 | 19.91 | 20.51 | 4023.46 | 6506.86 | 8623.71 | ✅ |
|
||||||
|
| c32_i2048_o128 | 32 | 2048 | 128 | 59.24 | 640 | 0 | 10.80 | 22125.82 | 1375.94 | 23501.76 | 367.53 | 672.56 | 1352.00 | 18.71 | 21.55 | 22.09 | 2732.47 | 3094.92 | 3404.73 | ✅ |
|
||||||
|
| c32_i2048_o2048 | 32 | 2048 | 2048 | 89.13 | 640 | 0 | 7.18 | 14705.49 | 1556.10 | 16261.59 | 199.91 | 291.10 | 1361.13 | 17.87 | 20.28 | 20.98 | 4056.53 | 6825.74 | 9339.78 | ✅ |
|
||||||
|
| c32_i2048_o256 | 32 | 2048 | 256 | 83.40 | 640 | 0 | 7.67 | 15715.67 | 1544.12 | 17259.80 | 202.63 | 380.57 | 1355.98 | 18.19 | 20.18 | 20.73 | 3838.39 | 5110.33 | 5339.24 | ✅ |
|
||||||
|
| c32_i2048_o4096 | 32 | 2048 | 4096 | 88.69 | 640 | 0 | 7.22 | 14778.01 | 1520.45 | 16298.46 | 199.19 | 288.11 | 1360.76 | 18.06 | 20.41 | 20.84 | 3982.98 | 6383.30 | 9405.39 | ✅ |
|
||||||
|
| c32_i2048_o512 | 32 | 2048 | 512 | 90.84 | 640 | 0 | 7.05 | 14428.41 | 1530.32 | 15958.73 | 197.17 | 253.30 | 1349.46 | 17.88 | 20.25 | 21.02 | 4060.40 | 6700.48 | 9080.73 | ✅ |
|
||||||
|
| c32_i4096_o1024 | 32 | 4096 | 1024 | 113.81 | 640 | 0 | 5.62 | 23032.89 | 1212.69 | 24245.58 | 339.67 | 542.37 | 2422.62 | 22.72 | 26.10 | 27.21 | 5215.72 | 8904.34 | 13457.62 | ✅ |
|
||||||
|
| c32_i4096_o128 | 32 | 4096 | 128 | 85.66 | 640 | 0 | 7.47 | 30604.61 | 950.87 | 31555.48 | 556.18 | 1180.23 | 2449.13 | 27.29 | 31.52 | 34.82 | 4002.80 | 4613.66 | 5507.05 | ✅ |
|
||||||
|
| c32_i4096_o2048 | 32 | 4096 | 2048 | 114.27 | 640 | 0 | 5.60 | 22939.90 | 1189.49 | 24129.39 | 341.11 | 550.60 | 2435.46 | 22.78 | 26.02 | 27.73 | 5165.78 | 9243.12 | 13018.90 | ✅ |
|
||||||
|
| c32_i4096_o256 | 32 | 4096 | 256 | 106.21 | 640 | 0 | 6.03 | 24681.91 | 1178.61 | 25860.52 | 350.79 | 564.37 | 2447.99 | 23.55 | 26.71 | 28.32 | 4937.54 | 6734.16 | 7405.49 | ✅ |
|
||||||
|
| c32_i4096_o4096 | 32 | 4096 | 4096 | 114.94 | 640 | 0 | 5.57 | 22806.49 | 1192.22 | 23998.71 | 341.86 | 550.55 | 2448.39 | 22.83 | 26.16 | 27.23 | 5203.63 | 8545.93 | 12625.46 | ✅ |
|
||||||
|
| c32_i4096_o512 | 32 | 4096 | 512 | 112.20 | 640 | 0 | 5.70 | 23364.93 | 1200.40 | 24565.33 | 340.20 | 560.25 | 2385.06 | 22.99 | 26.29 | 27.73 | 5150.64 | 8529.09 | 12164.94 | ✅ |
|
||||||
|
| c32_i512_o256 | 32 | 512 | 256 | 78.27 | 640 | 0 | 8.18 | 4186.51 | 1690.88 | 5877.39 | 149.00 | 289.93 | 473.49 | 16.81 | 19.27 | 22.36 | 3610.39 | 4786.53 | 5755.87 | ✅ |
|
||||||
|
| c32_i65536_o1024 | 32 | 65536 | 1024 | 266.52 | 160 | 0 | 0.60 | 39343.10 | 131.31 | 39474.42 | 8656.40 | 33860.81 | 43057.06 | 171.14 | 244.27 | 281.95 | 46303.99 | 103438.83 | 127750.75 | ❌ |
|
||||||
|
| c4_i512_o256 | 4 | 512 | 256 | 43.40 | 80 | 0 | 1.84 | 943.85 | 379.18 | 1323.03 | 118.07 | 125.54 | 241.46 | 7.83 | 8.29 | 8.68 | 1720.27 | 2165.16 | 2253.32 | ✅ |
|
||||||
|
| c64_i512_o256 | 64 | 512 | 256 | 109.33 | 1280 | 0 | 11.71 | 5994.26 | 2402.36 | 8396.62 | 171.45 | 299.06 | 771.51 | 24.39 | 26.72 | 27.58 | 5151.60 | 6720.29 | 6878.42 | ✅ |
|
||||||
|
| c8_i1024_o1024 | 8 | 1024 | 1024 | 47.80 | 160 | 0 | 3.35 | 3427.67 | 696.10 | 4123.77 | 129.12 | 192.22 | 281.04 | 9.27 | 9.95 | 10.24 | 2052.01 | 2788.77 | 3052.14 | ✅ |
|
||||||
|
| c8_i1024_o128 | 8 | 1024 | 128 | 31.03 | 160 | 0 | 5.16 | 5279.38 | 659.92 | 5939.30 | 187.55 | 269.21 | 292.69 | 8.88 | 9.90 | 9.97 | 1315.49 | 1387.68 | 1458.94 | ✅ |
|
||||||
|
| c8_i1024_o2048 | 8 | 1024 | 2048 | 47.63 | 160 | 0 | 3.36 | 3439.84 | 684.76 | 4124.59 | 139.64 | 191.21 | 283.85 | 9.49 | 10.93 | 11.77 | 2064.28 | 2867.60 | 3177.60 | ✅ |
|
||||||
|
| c8_i1024_o256 | 8 | 1024 | 256 | 47.68 | 160 | 0 | 3.36 | 3436.34 | 684.94 | 4121.28 | 134.22 | 195.34 | 290.76 | 9.44 | 10.83 | 11.66 | 2050.72 | 2669.20 | 2870.35 | ✅ |
|
||||||
|
| c8_i1024_o4096 | 8 | 1024 | 4096 | 48.02 | 160 | 0 | 3.33 | 3411.81 | 683.42 | 4095.23 | 132.59 | 194.92 | 285.39 | 9.39 | 10.45 | 11.05 | 2050.12 | 2647.51 | 2950.32 | ✅ |
|
||||||
|
| c8_i1024_o512 | 8 | 1024 | 512 | 48.89 | 160 | 0 | 3.27 | 3351.35 | 699.85 | 4051.20 | 128.43 | 130.82 | 286.17 | 9.25 | 9.96 | 10.23 | 2097.31 | 2846.33 | 3059.68 | ✅ |
|
||||||
|
| c8_i131072_o512 | 8 | 131072 | 512 | 593.71 | 160 | 0 | 0.27 | 35322.95 | 49.97 | 35372.92 | 8837.26 | 12856.65 | 21274.84 | 95.98 | 174.74 | 250.31 | 26223.81 | 43791.96 | 52427.10 | ❌ |
|
||||||
|
| c8_i16384_o1024 | 8 | 16384 | 1024 | 117.03 | 160 | 0 | 1.37 | 22399.35 | 381.47 | 22780.82 | 880.55 | 1325.49 | 2519.24 | 15.14 | 18.88 | 22.89 | 5043.96 | 9664.45 | 11942.89 | ✅ |
|
||||||
|
| c8_i16384_o128 | 8 | 16384 | 128 | 83.84 | 160 | 0 | 1.91 | 31265.56 | 241.22 | 31506.78 | 1766.89 | 2637.47 | 2676.54 | 14.95 | 23.25 | 26.08 | 3640.27 | 3713.62 | 5332.84 | ✅ |
|
||||||
|
| c8_i16384_o2048 | 8 | 16384 | 2048 | 111.65 | 160 | 0 | 1.43 | 23479.38 | 353.00 | 23832.38 | 858.33 | 1105.07 | 2515.80 | 15.80 | 19.89 | 21.08 | 4778.32 | 8910.12 | 13501.67 | ✅ |
|
||||||
|
| c8_i16384_o256 | 8 | 16384 | 256 | 101.77 | 160 | 0 | 1.57 | 25759.71 | 329.17 | 26088.88 | 867.77 | 1267.33 | 2511.92 | 17.06 | 21.93 | 23.13 | 4427.86 | 5958.68 | 6667.56 | ✅ |
|
||||||
|
| c8_i16384_o512 | 8 | 16384 | 512 | 109.64 | 160 | 0 | 1.46 | 23908.84 | 355.69 | 24264.53 | 908.35 | 1361.76 | 2515.33 | 15.82 | 20.35 | 22.15 | 4755.66 | 8684.67 | 10122.99 | ✅ |
|
||||||
|
| c8_i2048_o1024 | 8 | 2048 | 1024 | 53.11 | 160 | 0 | 3.01 | 6170.36 | 677.84 | 6848.20 | 150.15 | 153.04 | 410.55 | 9.49 | 10.24 | 10.65 | 2286.09 | 3870.04 | 5178.54 | ✅ |
|
||||||
|
| c8_i2048_o128 | 8 | 2048 | 128 | 33.35 | 160 | 0 | 4.80 | 9826.22 | 612.40 | 10438.62 | 228.93 | 388.77 | 404.46 | 9.33 | 10.33 | 11.02 | 1411.23 | 1530.95 | 1595.88 | ✅ |
|
||||||
|
| c8_i2048_o2048 | 8 | 2048 | 2048 | 51.12 | 160 | 0 | 3.13 | 6409.90 | 672.70 | 7082.60 | 150.70 | 165.44 | 418.00 | 9.56 | 10.25 | 10.68 | 2202.49 | 3406.00 | 4277.62 | ✅ |
|
||||||
|
| c8_i2048_o256 | 8 | 2048 | 256 | 48.03 | 160 | 0 | 3.33 | 6822.11 | 658.85 | 7480.96 | 148.57 | 147.18 | 403.31 | 9.66 | 10.48 | 10.98 | 2051.77 | 2715.99 | 2745.12 | ✅ |
|
||||||
|
| c8_i2048_o4096 | 8 | 2048 | 4096 | 53.12 | 160 | 0 | 3.01 | 6168.50 | 661.05 | 6829.55 | 150.99 | 219.64 | 421.18 | 9.58 | 10.44 | 10.58 | 2250.24 | 3788.73 | 4649.51 | ✅ |
|
||||||
|
| c8_i2048_o512 | 8 | 2048 | 512 | 50.97 | 160 | 0 | 3.14 | 6428.69 | 669.04 | 7097.73 | 148.41 | 147.71 | 403.41 | 9.60 | 10.48 | 11.08 | 2190.12 | 3392.37 | 4269.13 | ✅ |
|
||||||
|
| c8_i262144_o128 | 8 | 262144 | 128 | 1333.56 | 160 | 0 | 0.12 | 31451.94 | 12.61 | 31464.55 | 36128.45 | 57997.25 | 58592.27 | 215.95 | 392.14 | 446.94 | 59143.65 | 61777.27 | 74358.75 | ❌ |
|
||||||
|
| c8_i4096_o1024 | 8 | 4096 | 1024 | 60.18 | 160 | 0 | 2.66 | 10890.64 | 578.38 | 11469.02 | 253.10 | 364.29 | 731.93 | 10.57 | 11.89 | 12.17 | 2550.26 | 4642.49 | 7535.89 | ✅ |
|
||||||
|
| c8_i4096_o128 | 8 | 4096 | 128 | 40.35 | 160 | 0 | 3.96 | 16240.24 | 506.02 | 16746.26 | 376.51 | 695.40 | 727.26 | 10.62 | 11.99 | 13.73 | 1721.35 | 1910.10 | 2181.08 | ✅ |
|
||||||
|
| c8_i4096_o2048 | 8 | 4096 | 2048 | 57.14 | 160 | 0 | 2.80 | 11470.28 | 593.22 | 12063.50 | 254.35 | 361.27 | 732.14 | 10.52 | 11.68 | 12.36 | 2478.80 | 3843.55 | 5407.45 | ✅ |
|
||||||
|
| c8_i4096_o256 | 8 | 4096 | 256 | 54.73 | 160 | 0 | 2.92 | 11973.53 | 575.24 | 12548.76 | 252.76 | 366.29 | 732.21 | 10.63 | 12.02 | 12.59 | 2342.56 | 3122.99 | 3247.98 | ✅ |
|
||||||
|
| c8_i4096_o4096 | 8 | 4096 | 4096 | 61.97 | 160 | 0 | 2.58 | 10574.93 | 602.60 | 11177.53 | 251.62 | 276.31 | 733.83 | 10.48 | 11.81 | 12.23 | 2715.50 | 3797.26 | 5487.12 | ✅ |
|
||||||
|
| c8_i4096_o512 | 8 | 4096 | 512 | 57.59 | 160 | 0 | 2.78 | 11379.47 | 586.75 | 11966.22 | 255.02 | 376.23 | 733.95 | 10.54 | 11.70 | 12.17 | 2482.90 | 4048.32 | 5636.75 | ✅ |
|
||||||
|
| c8_i512_o256 | 8 | 512 | 256 | 48.34 | 160 | 0 | 3.31 | 1694.53 | 687.00 | 2381.52 | 125.68 | 194.69 | 258.22 | 9.43 | 10.43 | 10.95 | 2075.15 | 2669.68 | 2871.40 | ✅ |
|
||||||
|
| c8_i65536_o1024 | 8 | 65536 | 1024 | 298.66 | 160 | 0 | 0.54 | 35109.32 | 107.56 | 35216.88 | 3847.52 | 5884.53 | 9844.52 | 46.16 | 82.22 | 101.31 | 13001.39 | 21336.69 | 30533.51 | ⚠️ |
|
||||||
|
| c8_i65536_o128 | 8 | 65536 | 128 | 274.11 | 160 | 0 | 0.58 | 38254.29 | 70.30 | 38324.59 | 7223.66 | 11101.75 | 11262.28 | 40.50 | 72.54 | 72.63 | 12089.84 | 12347.21 | 12881.32 | ⚠️ |
|
||||||
|
| c8_i65536_o256 | 8 | 65536 | 256 | 291.01 | 160 | 0 | 0.55 | 36032.88 | 98.87 | 36131.75 | 3804.89 | 5718.15 | 9852.36 | 50.92 | 78.36 | 96.49 | 12843.24 | 18584.33 | 21293.29 | ❌ |
|
||||||
|
| c8_i65536_o512 | 8 | 65536 | 512 | 297.04 | 160 | 0 | 0.54 | 35301.20 | 111.90 | 35413.10 | 3902.29 | 6055.87 | 9835.60 | 44.23 | 73.93 | 83.82 | 13139.03 | 22983.14 | 31590.46 | ⚠️ |
|
||||||
|
|
||||||
|
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1,126 +0,0 @@
|
|||||||
# dsv4_h200_vllm_tp4_custom_bench_no_fp8
|
|
||||||
|
|
||||||
## 目的
|
|
||||||
|
|
||||||
当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU：
|
|
||||||
|
|
||||||
> **注意**：本实验变体 **不启用 `--kv-cache-dtype fp8`**，用于对比 FP8 KV Cache 与默认精度的性能差异。
|
|
||||||
|
|
||||||
| 服务 | 端口 | GPU 组 |
|
|
||||||
|------|------|--------|
|
|
||||||
| 1 | 30005 | 0, 1, 2, 3 |
|
|
||||||
| 2 | 30006 | 4, 5, 6, 7 |
|
|
||||||
|
|
||||||
相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。
|
|
||||||
|
|
||||||
## 与 TP=2 方案的区别
|
|
||||||
|
|
||||||
| 特性 | TP=2 (4 服务) | TP=4 (2 服务) |
|
|
||||||
|------|---------------|---------------|
|
|
||||||
| 服务数量 | 4 | 2 |
|
|
||||||
| 每服务 GPU | 2 | 4 |
|
|
||||||
| 单服务吞吐 | 较低 | 较高 |
|
|
||||||
| 集群总吞吐 | 相近 | 相近 |
|
|
||||||
| 适用场景 | 低并发、低延迟 | 高并发、大 batch |
|
|
||||||
|
|
||||||
## 文件说明
|
|
||||||
|
|
||||||
| 文件 | 说明 |
|
|
||||||
|------|------|
|
|
||||||
| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） |
|
|
||||||
| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） |
|
|
||||||
| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 |
|
|
||||||
| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） |
|
|
||||||
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
|
|
||||||
|
|
||||||
## 快速运行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 完整流程（自动启动 2 个服务、压测、生成报告）
|
|
||||||
bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 跳过服务管理（已有 2 个服务在运行）
|
|
||||||
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 单独运行压测客户端（多服务，带健康检查）
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports 30005,30006 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
|
|
||||||
--lb-strategy round_robin \
|
|
||||||
--health-check-interval 5 \
|
|
||||||
--health-max-failures 2 \
|
|
||||||
--health-recovery-interval 10 \
|
|
||||||
--request-max-retries 2 \
|
|
||||||
--output-file /tmp/test.jsonl
|
|
||||||
|
|
||||||
# 单独运行压测客户端（单服务，向后兼容）
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \
|
|
||||||
--output-file /tmp/test_single.jsonl
|
|
||||||
|
|
||||||
# 解析已有结果
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \
|
|
||||||
experiments/dsv4_h200_vllm_tp4_custom_bench/results/<RUN_ID>
|
|
||||||
```
|
|
||||||
|
|
||||||
## 场景设计
|
|
||||||
|
|
||||||
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
|
|
||||||
|
|
||||||
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|
|
||||||
|------|----------|----------|--------|------|
|
|
||||||
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
|
|
||||||
| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req |
|
|
||||||
| 4 | 512 | 256 | 80 | 中低并发 |
|
|
||||||
| 8 | 512 | 256 | 160 | 中等并发 |
|
|
||||||
| 16 | 512 | 256 | 320 | 中高并发 |
|
|
||||||
| 32 | 512 | 256 | 640 | 高并发 |
|
|
||||||
| 64 | 512 | 256 | 1280 | 很高并发 |
|
|
||||||
| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 |
|
|
||||||
| 1 | 2048 | 512 | 20 | 长输入基线 |
|
|
||||||
| 8 | 2048 | 512 | 160 | 长输入中并发 |
|
|
||||||
| 32 | 2048 | 512 | 640 | 长输入高并发 |
|
|
||||||
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
|
|
||||||
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
|
|
||||||
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
|
|
||||||
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
|
|
||||||
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
|
|
||||||
|
|
||||||
## 健康检查与故障自动跳过
|
|
||||||
|
|
||||||
同 TP=2 版本，详见原 README。
|
|
||||||
|
|
||||||
## 负载均衡策略
|
|
||||||
|
|
||||||
同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。
|
|
||||||
|
|
||||||
## 输出格式
|
|
||||||
|
|
||||||
同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。
|
|
||||||
|
|
||||||
## 单独启动/停止 2 个服务
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 启动 2 个服务
|
|
||||||
bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh
|
|
||||||
|
|
||||||
# 停止（run_bench.sh 会自动停止，但也可以手动）
|
|
||||||
for port in 30005 30006; do
|
|
||||||
pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid"
|
|
||||||
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
|
|
||||||
rm -f "$pid_file"
|
|
||||||
done
|
|
||||||
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。
|
|
||||||
- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。
|
|
||||||
- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。
|
|
||||||
- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。
|
|
||||||
- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。
|
|
||||||
@ -1,617 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
|
|
||||||
|
|
||||||
Sends concurrent requests to multiple inference servers and records per-request
|
|
||||||
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
|
|
||||||
|
|
||||||
Designed for TP=2 on 8-GPU setups where 4 independent services run on
|
|
||||||
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
|
|
||||||
round-robin or random load balancing. Unhealthy services are automatically
|
|
||||||
skipped with periodic retry.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# Single service (backward compatible)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
# Multi-service (4 services on ports 30005-30008)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports 30005,30006,30007,30008 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--lb-strategy round_robin \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
The output JSONL contains one object per request with raw timing data.
|
|
||||||
A final summary line ("completed" field) aggregates all requests.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import random
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from dataclasses import asdict, dataclass, field
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
# Try aiohttp first; fall back to urllib for stdlib-only environments.
|
|
||||||
try:
|
|
||||||
import aiohttp
|
|
||||||
HAS_AIOHTTP = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_AIOHTTP = False
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Data structures
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class RequestResult:
|
|
||||||
request_id: int
|
|
||||||
input_tokens: int = 0
|
|
||||||
output_tokens: int = 0
|
|
||||||
first_token_time_ms: float = 0.0 # TTFT
|
|
||||||
e2e_latency_ms: float = 0.0 # total wall time
|
|
||||||
inter_token_latencies: list[float] = field(default_factory=list)
|
|
||||||
success: bool = False
|
|
||||||
error: str = ""
|
|
||||||
target_port: int = 0 # which service handled this request
|
|
||||||
retry_count: int = 0 # how many times this request was retried
|
|
||||||
|
|
||||||
@property
|
|
||||||
def tpot_ms(self) -> float:
|
|
||||||
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
|
|
||||||
if self.output_tokens <= 1:
|
|
||||||
return 0.0
|
|
||||||
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def mean_itl_ms(self) -> float:
|
|
||||||
if not self.inter_token_latencies:
|
|
||||||
return 0.0
|
|
||||||
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
|
|
||||||
|
|
||||||
def to_dict(self) -> dict[str, Any]:
|
|
||||||
d = asdict(self)
|
|
||||||
d["tpot_ms"] = self.tpot_ms
|
|
||||||
d["mean_itl_ms"] = self.mean_itl_ms
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Health checker
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class HealthChecker:
|
|
||||||
"""Periodically check service health and maintain a healthy-port list."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
check_interval: float = 5.0,
|
|
||||||
max_failures: int = 2,
|
|
||||||
recovery_interval: float = 10.0,
|
|
||||||
):
|
|
||||||
self.host = host
|
|
||||||
self.all_ports = ports
|
|
||||||
self.check_interval = check_interval
|
|
||||||
self.max_failures = max_failures
|
|
||||||
self.recovery_interval = recovery_interval
|
|
||||||
|
|
||||||
# port -> consecutive failure count
|
|
||||||
self._failures: dict[int, int] = {p: 0 for p in ports}
|
|
||||||
# port -> last healthy timestamp
|
|
||||||
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
self._task: asyncio.Task | None = None
|
|
||||||
|
|
||||||
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
|
|
||||||
url = f"http://{self.host}:{port}/health"
|
|
||||||
try:
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
else:
|
|
||||||
req = urllib.request.Request(url, method="GET")
|
|
||||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
checks = [self._check_one(p, session) for p in self.all_ports]
|
|
||||||
results = await asyncio.gather(*checks, return_exceptions=True)
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
for port, ok in zip(self.all_ports, results):
|
|
||||||
if isinstance(ok, Exception):
|
|
||||||
ok = False
|
|
||||||
if ok:
|
|
||||||
self._failures[port] = 0
|
|
||||||
self._last_healthy[port] = now
|
|
||||||
else:
|
|
||||||
self._failures[port] += 1
|
|
||||||
|
|
||||||
async def start(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
async def _loop() -> None:
|
|
||||||
while True:
|
|
||||||
await asyncio.sleep(self.check_interval)
|
|
||||||
await self._check_all(session)
|
|
||||||
self._task = asyncio.create_task(_loop())
|
|
||||||
|
|
||||||
async def stop(self) -> None:
|
|
||||||
if self._task is not None:
|
|
||||||
self._task.cancel()
|
|
||||||
try:
|
|
||||||
await self._task
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
async def healthy_ports(self) -> list[int]:
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
healthy = []
|
|
||||||
for p in self.all_ports:
|
|
||||||
if self._failures[p] < self.max_failures:
|
|
||||||
healthy.append(p)
|
|
||||||
elif now - self._last_healthy[p] > self.recovery_interval:
|
|
||||||
# Give it another chance after recovery_interval.
|
|
||||||
self._failures[p] = max(0, self._failures[p] - 1)
|
|
||||||
healthy.append(p)
|
|
||||||
return healthy
|
|
||||||
|
|
||||||
async def is_healthy(self, port: int) -> bool:
|
|
||||||
ports = await self.healthy_ports
|
|
||||||
return port in ports
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Load balancer with failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class LoadBalancer:
|
|
||||||
"""Distribute requests across healthy backend ports with auto-failover."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
ports: list[int],
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
strategy: str = "round_robin",
|
|
||||||
):
|
|
||||||
self.all_ports = ports
|
|
||||||
self.health_checker = health_checker
|
|
||||||
self.strategy = strategy
|
|
||||||
self._idx = 0
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
|
|
||||||
def _next_round_robin(self, healthy: list[int]) -> int:
|
|
||||||
if not healthy:
|
|
||||||
return self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
for _ in range(len(self.all_ports)):
|
|
||||||
port = self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
self._idx += 1
|
|
||||||
if port in healthy:
|
|
||||||
return port
|
|
||||||
# Fallback: all unhealthy, pick first healthy anyway.
|
|
||||||
return healthy[0] if healthy else self.all_ports[0]
|
|
||||||
|
|
||||||
def _next_random(self, healthy: list[int]) -> int:
|
|
||||||
if healthy:
|
|
||||||
return random.choice(healthy)
|
|
||||||
return random.choice(self.all_ports)
|
|
||||||
|
|
||||||
async def next_port(self) -> int:
|
|
||||||
healthy = await self.health_checker.healthy_ports
|
|
||||||
async with self._lock:
|
|
||||||
if self.strategy == "round_robin":
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
elif self.strategy in ("random", "least_conn"):
|
|
||||||
return self._next_random(healthy)
|
|
||||||
else:
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def port_count(self) -> int:
|
|
||||||
return len(self.all_ports)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Token helpers
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
|
||||||
needed = max(token_len, 1)
|
|
||||||
tokens: list[str] = []
|
|
||||||
while len(tokens) < needed:
|
|
||||||
tokens.extend(words)
|
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# aiohttp-based async request (preferred)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_aiohttp(
|
|
||||||
session: aiohttp.ClientSession,
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": True,
|
|
||||||
}
|
|
||||||
headers = {"Content-Type": "application/json"}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
first_token_received = False
|
|
||||||
last_token_time = 0.0
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with session.post(url, json=payload, headers=headers) as resp:
|
|
||||||
if resp.status != 200:
|
|
||||||
text = await resp.text()
|
|
||||||
result.error = f"HTTP {resp.status}: {text[:200]}"
|
|
||||||
return result
|
|
||||||
|
|
||||||
async for line in resp.content:
|
|
||||||
line = line.decode("utf-8").strip()
|
|
||||||
if not line or not line.startswith("data: "):
|
|
||||||
continue
|
|
||||||
data_str = line[len("data: "):]
|
|
||||||
if data_str == "[DONE]":
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
chunk = json.loads(data_str)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
choices = chunk.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
continue
|
|
||||||
|
|
||||||
delta = choices[0].get("delta", {})
|
|
||||||
content = delta.get("content", "")
|
|
||||||
if content:
|
|
||||||
now = time.perf_counter()
|
|
||||||
if not first_token_received:
|
|
||||||
result.first_token_time_ms = (now - start_time) * 1000
|
|
||||||
first_token_received = True
|
|
||||||
else:
|
|
||||||
result.inter_token_latencies.append((now - last_token_time) * 1000)
|
|
||||||
last_token_time = now
|
|
||||||
result.output_tokens += 1
|
|
||||||
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.input_tokens = len(prompt.split()) # approximate
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
result.error = "timeout"
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# urllib-based synchronous request (fallback, no aiohttp)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def send_request_urllib(
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": False,
|
|
||||||
}
|
|
||||||
data = json.dumps(payload).encode("utf-8")
|
|
||||||
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
|
|
||||||
try:
|
|
||||||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
|
||||||
with urllib.request.urlopen(req, timeout=600) as resp:
|
|
||||||
body = json.loads(resp.read().decode("utf-8"))
|
|
||||||
|
|
||||||
choices = body.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
result.error = "no choices in response"
|
|
||||||
return result
|
|
||||||
|
|
||||||
content = choices[0].get("message", {}).get("content", "")
|
|
||||||
result.output_tokens = len(content.split()) if content else 0
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.first_token_time_ms = result.e2e_latency_ms
|
|
||||||
result.input_tokens = len(prompt.split())
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Unified send_request wrapper with retry & failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_with_retry(
|
|
||||||
session: aiohttp.ClientSession | None,
|
|
||||||
host: str,
|
|
||||||
lb: LoadBalancer,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
max_retries: int = 2,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Send a request, retrying on failure with a different port each time."""
|
|
||||||
result = RequestResult(request_id=request_id)
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
port = await lb.next_port()
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
result = await send_request_aiohttp(
|
|
||||||
session, host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
result = await loop.run_in_executor(
|
|
||||||
None, send_request_urllib,
|
|
||||||
host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
result.retry_count = attempt
|
|
||||||
if result.success:
|
|
||||||
return result
|
|
||||||
# If failed, try another port on next iteration (unless last attempt).
|
|
||||||
if attempt < max_retries:
|
|
||||||
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Benchmark orchestration
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def run_benchmark(
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
lb: LoadBalancer,
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
model: str,
|
|
||||||
concurrency: int,
|
|
||||||
input_len: int,
|
|
||||||
output_len: int,
|
|
||||||
num_prompts: int,
|
|
||||||
temperature: float,
|
|
||||||
output_file: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Run the benchmark and write a JSONL file."""
|
|
||||||
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
|
|
||||||
results: list[RequestResult] = []
|
|
||||||
semaphore = asyncio.Semaphore(concurrency)
|
|
||||||
|
|
||||||
start = time.perf_counter()
|
|
||||||
|
|
||||||
if HAS_AIOHTTP:
|
|
||||||
connector = aiohttp.TCPConnector(limit=concurrency * 2)
|
|
||||||
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
|
|
||||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
|
||||||
await health_checker.start(session)
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
else:
|
|
||||||
await health_checker.start(None)
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
None, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
|
|
||||||
duration = time.perf_counter() - start
|
|
||||||
|
|
||||||
# Write JSONL
|
|
||||||
with open(output_file, "w", encoding="utf-8") as f:
|
|
||||||
for r in results:
|
|
||||||
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
# Compute summary
|
|
||||||
successful = [r for r in results if r.success]
|
|
||||||
failed = len(results) - len(successful)
|
|
||||||
retried = [r for r in results if r.retry_count > 0]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
ttfts = [r.first_token_time_ms for r in successful]
|
|
||||||
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
|
|
||||||
e2es = [r.e2e_latency_ms for r in successful]
|
|
||||||
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
|
|
||||||
|
|
||||||
total_input_tokens = sum(r.input_tokens for r in successful)
|
|
||||||
total_output_tokens = sum(r.output_tokens for r in successful)
|
|
||||||
|
|
||||||
# Per-port breakdown
|
|
||||||
per_port_stats: dict[int, dict[str, Any]] = {}
|
|
||||||
for port in ports:
|
|
||||||
port_results = [r for r in successful if r.target_port == port]
|
|
||||||
port_failed = [r for r in results if r.target_port == port and not r.success]
|
|
||||||
if port_results or port_failed:
|
|
||||||
port_ttfts = [r.first_token_time_ms for r in port_results]
|
|
||||||
port_e2es = [r.e2e_latency_ms for r in port_results]
|
|
||||||
per_port_stats[port] = {
|
|
||||||
"count": len(port_results),
|
|
||||||
"failed": len(port_failed),
|
|
||||||
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
|
|
||||||
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
|
|
||||||
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Health-check stats
|
|
||||||
healthy_at_end = await health_checker.healthy_ports
|
|
||||||
|
|
||||||
summary = {
|
|
||||||
"completed": len(successful),
|
|
||||||
"failed": failed,
|
|
||||||
"retried": len(retried),
|
|
||||||
"duration": duration,
|
|
||||||
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
|
|
||||||
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
|
|
||||||
"total_input_tokens": total_input_tokens,
|
|
||||||
"total_output_tokens": total_output_tokens,
|
|
||||||
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
|
|
||||||
"median_ttft_ms": percentile(ttfts, 50),
|
|
||||||
"p90_ttft_ms": percentile(ttfts, 90),
|
|
||||||
"p95_ttft_ms": percentile(ttfts, 95),
|
|
||||||
"p99_ttft_ms": percentile(ttfts, 99),
|
|
||||||
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
|
|
||||||
"median_tpot_ms": percentile(tpots, 50),
|
|
||||||
"p90_tpot_ms": percentile(tpots, 90),
|
|
||||||
"p95_tpot_ms": percentile(tpots, 95),
|
|
||||||
"p99_tpot_ms": percentile(tpots, 99),
|
|
||||||
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
|
|
||||||
"median_e2e_latency_ms": percentile(e2es, 50),
|
|
||||||
"p90_e2e_latency_ms": percentile(e2es, 90),
|
|
||||||
"p95_e2e_latency_ms": percentile(e2es, 95),
|
|
||||||
"p99_e2e_latency_ms": percentile(e2es, 99),
|
|
||||||
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
|
|
||||||
"median_itl_ms": percentile(itls, 50),
|
|
||||||
"p90_itl_ms": percentile(itls, 90),
|
|
||||||
"p95_itl_ms": percentile(itls, 95),
|
|
||||||
"p99_itl_ms": percentile(itls, 99),
|
|
||||||
"per_port_stats": per_port_stats,
|
|
||||||
"healthy_ports_at_end": healthy_at_end,
|
|
||||||
"lb_strategy": lb.strategy,
|
|
||||||
"num_services": lb.port_count,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Append summary as the last line
|
|
||||||
with open(output_file, "a", encoding="utf-8") as f:
|
|
||||||
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
|
|
||||||
parser.add_argument("--host", default="127.0.0.1")
|
|
||||||
parser.add_argument("--port", type=int, default=30005,
|
|
||||||
help="Single service port (backward compatible)")
|
|
||||||
parser.add_argument("--ports", default="",
|
|
||||||
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
|
|
||||||
parser.add_argument("--model", default="deepseek-v4-flash")
|
|
||||||
parser.add_argument("--concurrency", type=int, default=1)
|
|
||||||
parser.add_argument("--input-len", type=int, default=512)
|
|
||||||
parser.add_argument("--output-len", type=int, default=256)
|
|
||||||
parser.add_argument("--num-prompts", type=int, default=10)
|
|
||||||
parser.add_argument("--temperature", type=float, default=0.0)
|
|
||||||
parser.add_argument("--lb-strategy", default="round_robin",
|
|
||||||
choices=["round_robin", "random", "least_conn"],
|
|
||||||
help="Load balancing strategy across services")
|
|
||||||
parser.add_argument("--health-check-interval", type=float, default=5.0,
|
|
||||||
help="Seconds between health checks (default: 5)")
|
|
||||||
parser.add_argument("--health-max-failures", type=int, default=2,
|
|
||||||
help="Consecutive failures before marking a port unhealthy (default: 2)")
|
|
||||||
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
|
|
||||||
help="Seconds before retrying an unhealthy port (default: 10)")
|
|
||||||
parser.add_argument("--request-max-retries", type=int, default=2,
|
|
||||||
help="Max retries per request on failure (default: 2)")
|
|
||||||
parser.add_argument("--output-file", required=True)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Determine ports: --ports takes precedence, else fall back to --port.
|
|
||||||
if args.ports:
|
|
||||||
ports = [int(p.strip()) for p in args.ports.split(",")]
|
|
||||||
else:
|
|
||||||
ports = [args.port]
|
|
||||||
|
|
||||||
health_checker = HealthChecker(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
check_interval=args.health_check_interval,
|
|
||||||
max_failures=args.health_max_failures,
|
|
||||||
recovery_interval=args.health_recovery_interval,
|
|
||||||
)
|
|
||||||
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
|
|
||||||
|
|
||||||
summary = asyncio.run(run_benchmark(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
lb=lb,
|
|
||||||
health_checker=health_checker,
|
|
||||||
model=args.model,
|
|
||||||
concurrency=args.concurrency,
|
|
||||||
input_len=args.input_len,
|
|
||||||
output_len=args.output_len,
|
|
||||||
num_prompts=args.num_prompts,
|
|
||||||
temperature=args.temperature,
|
|
||||||
output_file=args.output_file,
|
|
||||||
))
|
|
||||||
|
|
||||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,84 +0,0 @@
|
|||||||
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
|
|
||||||
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
|
|
||||||
#
|
|
||||||
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
|
|
||||||
# Service 1: port 30005, GPUs 0,1
|
|
||||||
# Service 2: port 30006, GPUs 2,3
|
|
||||||
# Service 3: port 30007, GPUs 4,5
|
|
||||||
# Service 4: port 30008, GPUs 6,7
|
|
||||||
#
|
|
||||||
# bench_client.py distributes requests across all 4 services via round-robin
|
|
||||||
# load balancing, giving accurate cluster-wide throughput even at low
|
|
||||||
# per-service concurrency.
|
|
||||||
|
|
||||||
EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench_no_fp8"
|
|
||||||
MODEL_NAME="DeepSeek-V4-Flash"
|
|
||||||
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
||||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
||||||
|
|
||||||
# Primary port (backward compatible); multi-service ports are in PORTS.
|
|
||||||
PORT="30005"
|
|
||||||
PORTS="30005,30006"
|
|
||||||
|
|
||||||
BACKEND="vllm"
|
|
||||||
ENGINE="vllm"
|
|
||||||
|
|
||||||
# Native virtual environments on the host.
|
|
||||||
VENV_SERVER="/data/user1/yy/envs/vllm"
|
|
||||||
VENV_CLIENT="/data/user1/yy/envs/vllm"
|
|
||||||
|
|
||||||
# Load balancing strategy: round_robin | random | least_conn
|
|
||||||
LB_STRATEGY="round_robin"
|
|
||||||
|
|
||||||
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
|
||||||
# For multi-service we include low-concurrency scenarios to show how
|
|
||||||
# requests spread across 2 services.
|
|
||||||
SCENARIOS=(
|
|
||||||
"1 512 256 20"
|
|
||||||
"2 512 256 40"
|
|
||||||
"4 512 256 80"
|
|
||||||
"8 512 256 160"
|
|
||||||
"16 512 256 320"
|
|
||||||
"32 512 256 640"
|
|
||||||
"64 512 256 1280"
|
|
||||||
"128 512 256 2560"
|
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
|
||||||
"32 2048 512 640"
|
|
||||||
"128 2048 512 2560"
|
|
||||||
"1 4096 1024 20"
|
|
||||||
"8 4096 1024 160"
|
|
||||||
"32 4096 1024 640"
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
|
||||||
"8 8192 1024 160"
|
|
||||||
"32 8192 1024 640"
|
|
||||||
"128 8192 1024 2560"
|
|
||||||
"1 16384 1024 20"
|
|
||||||
"8 16384 1024 160"
|
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
|
||||||
"1 32768 1024 20"
|
|
||||||
"8 32768 1024 160"
|
|
||||||
"32 32768 1024 640"
|
|
||||||
"128 32768 1024 2560"
|
|
||||||
"1 65536 1024 20"
|
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
|
||||||
"128 65536 1024 2560"
|
|
||||||
"1 131072 1024 20"
|
|
||||||
"8 131072 1024 160"
|
|
||||||
"32 131072 1024 640"
|
|
||||||
"128 131072 1024 2560"
|
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
|
||||||
"32 262144 1024 640"
|
|
||||||
"128 262144 1024 2560"
|
|
||||||
"1 524288 1024 20"
|
|
||||||
"8 524288 1024 160"
|
|
||||||
"32 524288 1024 640"
|
|
||||||
"128 524288 1024 2560"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
|
||||||
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"
|
|
||||||
@ -1,206 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
|
|
||||||
|
|
||||||
Reads JSONL files produced by bench_client.py (one request per line,
|
|
||||||
last line is the summary) and generates:
|
|
||||||
- results.json (appended scenarios, following the standard schema)
|
|
||||||
- report.md (human-readable summary)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python3 parse_results.py <result_root>
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
|
|
||||||
"""Read the summary JSON (last line) and all per-request lines."""
|
|
||||||
requests: list[dict] = []
|
|
||||||
summary: dict | None = None
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
for line in f:
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
obj = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
# The summary line has "completed" and "duration" keys.
|
|
||||||
if "completed" in obj and "duration" in obj:
|
|
||||||
summary = obj
|
|
||||||
else:
|
|
||||||
requests.append(obj)
|
|
||||||
return summary, requests
|
|
||||||
|
|
||||||
|
|
||||||
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
|
|
||||||
completed = summary.get("completed", 0)
|
|
||||||
total = len(requests)
|
|
||||||
failed = total - completed if total > 0 else 0
|
|
||||||
duration_s = summary.get("duration", 0.0)
|
|
||||||
|
|
||||||
# Extract per-request metrics for percentiles.
|
|
||||||
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
|
|
||||||
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
|
|
||||||
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
|
|
||||||
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": completed,
|
|
||||||
"failed": failed,
|
|
||||||
"duration_s": duration_s,
|
|
||||||
"request_throughput": summary.get("request_throughput", 0.0),
|
|
||||||
"input_token_throughput": summary.get("input_throughput", 0.0),
|
|
||||||
"output_token_throughput": summary.get("output_throughput", 0.0),
|
|
||||||
"total_token_throughput": summary.get("total_throughput", 0.0),
|
|
||||||
"total_input_tokens": summary.get("total_input_tokens", 0),
|
|
||||||
"total_output_tokens": summary.get("total_output_tokens", 0),
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": summary.get("mean_e2e_latency_ms", 0.0),
|
|
||||||
"p50": percentile(e2es, 50),
|
|
||||||
"p90": percentile(e2es, 90),
|
|
||||||
"p95": percentile(e2es, 95),
|
|
||||||
"p99": percentile(e2es, 99),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": summary.get("mean_ttft_ms", 0.0),
|
|
||||||
"p50": percentile(ttfts, 50),
|
|
||||||
"p90": percentile(ttfts, 90),
|
|
||||||
"p95": percentile(ttfts, 95),
|
|
||||||
"p99": percentile(ttfts, 99),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": summary.get("mean_tpot_ms", 0.0),
|
|
||||||
"p50": percentile(tpots, 50),
|
|
||||||
"p90": percentile(tpots, 90),
|
|
||||||
"p95": percentile(tpots, 95),
|
|
||||||
"p99": percentile(tpots, 99),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": summary.get("mean_itl_ms", 0.0),
|
|
||||||
"p50": percentile(itls, 50),
|
|
||||||
"p90": percentile(itls, 90),
|
|
||||||
"p95": percentile(itls, 95),
|
|
||||||
"p99": percentile(itls, 99),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
|
||||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
|
||||||
|
|
||||||
|
|
||||||
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
|
|
||||||
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
|
|
||||||
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
|
|
||||||
if ttft_ok and tpot_ok:
|
|
||||||
mark = "✅"
|
|
||||||
elif ttft_ok or tpot_ok:
|
|
||||||
mark = "⚠️"
|
|
||||||
else:
|
|
||||||
mark = "❌"
|
|
||||||
return {
|
|
||||||
"ttft_p95_ok": ttft_ok,
|
|
||||||
"tpot_mean_ok": tpot_ok,
|
|
||||||
"overall": mark,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
|
||||||
with open(results_json, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
data["scenarios"].append(scenario)
|
|
||||||
with open(results_json, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
|
|
||||||
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
|
|
||||||
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
|
|
||||||
f.write(f"- **Result root**: `{result_root}`\n\n")
|
|
||||||
|
|
||||||
f.write("## Results\n\n")
|
|
||||||
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
|
||||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
|
||||||
|
|
||||||
for s in scenarios:
|
|
||||||
cfg = s["config"]
|
|
||||||
m = s["metrics"]
|
|
||||||
slo = s.get("slo_status", {}).get("overall", "")
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
|
||||||
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
|
|
||||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
|
||||||
f"{m['total_token_throughput']:.2f} | "
|
|
||||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
|
||||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
|
||||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
|
||||||
)
|
|
||||||
f.write("\n")
|
|
||||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
results_json = result_root / "results.json"
|
|
||||||
|
|
||||||
if not raw_dir.exists():
|
|
||||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
|
|
||||||
parts = jsonl_path.stem.split("_")
|
|
||||||
if len(parts) < 5:
|
|
||||||
continue
|
|
||||||
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
|
|
||||||
|
|
||||||
summary, requests = parse_jsonl(jsonl_path)
|
|
||||||
if summary is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
metrics = compute_metrics(summary, requests)
|
|
||||||
scenario = {
|
|
||||||
"name": scenario_name(concurrency, input_len, output_len),
|
|
||||||
"config": {
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"dataset": "random",
|
|
||||||
"num_prompts": metrics["success"] + metrics["failed"],
|
|
||||||
},
|
|
||||||
"metrics": metrics,
|
|
||||||
"slo_status": slo_status(metrics),
|
|
||||||
"raw_file": str(jsonl_path),
|
|
||||||
}
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
if not scenarios:
|
|
||||||
print("No benchmark outputs found to parse")
|
|
||||||
return
|
|
||||||
|
|
||||||
if results_json.exists():
|
|
||||||
for s in scenarios:
|
|
||||||
append_scenario(results_json, s)
|
|
||||||
|
|
||||||
generate_report(result_root, scenarios)
|
|
||||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,278 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200.
|
|
||||||
#
|
|
||||||
# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7).
|
|
||||||
# bench_client.py distributes requests across all 2 services via round-robin LB,
|
|
||||||
# giving accurate cluster-wide throughput even at low per-service concurrency.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# bash run_bench.sh
|
|
||||||
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
|
||||||
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
||||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
|
||||||
RAW_DIR="${RESULT_ROOT}/raw_outputs"
|
|
||||||
LOG_DIR="${RESULT_ROOT}/logs"
|
|
||||||
|
|
||||||
ensure_result_root "$RESULT_ROOT"
|
|
||||||
log_init "${LOG_DIR}/orchestrator.log"
|
|
||||||
|
|
||||||
log "experiment=${EXPERIMENT_NAME}"
|
|
||||||
log "run_id=${RUN_ID}"
|
|
||||||
log "result_root=${RESULT_ROOT}"
|
|
||||||
log "platform=${PLATFORM}"
|
|
||||||
log "chip=${CHIP}"
|
|
||||||
log "accelerator=${ACCELERATOR}"
|
|
||||||
log "engine=${ENGINE}"
|
|
||||||
log "hardware=${HARDWARE}"
|
|
||||||
log "model=${MODEL_PATH}"
|
|
||||||
log "services=${PORTS}"
|
|
||||||
log "lb_strategy=${LB_STRATEGY}"
|
|
||||||
log "server_start_script=${SERVER_START_SCRIPT}"
|
|
||||||
|
|
||||||
# Write initial metadata for this run.
|
|
||||||
METADATA_JSON="${RESULT_ROOT}/results.json"
|
|
||||||
write_metadata_json \
|
|
||||||
"$METADATA_JSON" \
|
|
||||||
"$EXPERIMENT_NAME" \
|
|
||||||
"$RUN_ID" \
|
|
||||||
"$MODEL_PATH" \
|
|
||||||
"$BACKEND" \
|
|
||||||
"$ENGINE" \
|
|
||||||
"$HARDWARE" \
|
|
||||||
"$ACCELERATOR" \
|
|
||||||
"$CHIP" \
|
|
||||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
|
||||||
"$VENV_SERVER" \
|
|
||||||
"H200 2x vLLM TP=4 multi-service benchmark using bench_client.py (no FP8 KV cache)"
|
|
||||||
|
|
||||||
# Update metadata with config.
|
|
||||||
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
|
|
||||||
import json, sys
|
|
||||||
path = sys.argv[1]
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
data["config"] = {
|
|
||||||
"tp": 4,
|
|
||||||
"num_services": 2,
|
|
||||||
"ports": "30005,30006",
|
|
||||||
"gpu_groups": "0,1,2,3|4,5,6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20", "2 512 256 40", "4 512 256 80",
|
|
||||||
"8 512 256 160", "16 512 256 320", "32 512 256 640",
|
|
||||||
"64 512 256 1280", "128 512 256 2560",
|
|
||||||
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
|
|
||||||
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
with open(path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
PY
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Server helpers (multi-service)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
is_server_healthy() {
|
|
||||||
local port="$1"
|
|
||||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
are_all_services_healthy() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if ! is_server_healthy "$p"; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_all_servers() {
|
|
||||||
log "stopping all vllm services"
|
|
||||||
# Kill by PID files first.
|
|
||||||
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
|
|
||||||
[[ -f "$pid_file" ]] || continue
|
|
||||||
local pid
|
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "killing pid=${pid}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
done
|
|
||||||
# Fallback: kill any remaining vllm processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
}
|
|
||||||
|
|
||||||
start_all_servers() {
|
|
||||||
log "starting all vllm services with ${SERVER_START_SCRIPT}"
|
|
||||||
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
|
|
||||||
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
|
|
||||||
|
|
||||||
if ! are_all_services_healthy "$PORTS"; then
|
|
||||||
log "error: not all vllm services became healthy"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
log "all vllm services are healthy"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Pre-benchmark health check with detailed reporting
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_services_detailed() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy_count=0
|
|
||||||
local unhealthy_ports=""
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
((healthy_count++))
|
|
||||||
else
|
|
||||||
unhealthy_ports="${unhealthy_ports}${p} "
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo "$healthy_count"
|
|
||||||
if [[ -n "$unhealthy_ports" ]]; then
|
|
||||||
echo "unhealthy: ${unhealthy_ports}" >&2
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_all_services() {
|
|
||||||
local ports="$1"
|
|
||||||
local max_wait="${2:-240}"
|
|
||||||
local interval="${3:-5}"
|
|
||||||
local elapsed=0
|
|
||||||
while (( elapsed < max_wait )); do
|
|
||||||
local healthy_count
|
|
||||||
healthy_count="$(check_services_detailed "$ports" | head -1)"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
if (( healthy_count == ${#PORT_LIST[@]} )); then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
|
|
||||||
sleep "$interval"
|
|
||||||
((elapsed += interval))
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Mid-benchmark health check (called between scenarios)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_and_report_services() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy=()
|
|
||||||
local unhealthy=()
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
healthy+=("$p")
|
|
||||||
else
|
|
||||||
unhealthy+=("$p")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
|
|
||||||
if [[ ${#unhealthy[@]} -gt 0 ]]; then
|
|
||||||
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
|
|
||||||
fi
|
|
||||||
return ${#unhealthy[@]}
|
|
||||||
}
|
|
||||||
|
|
||||||
on_exit() {
|
|
||||||
local code=$?
|
|
||||||
log "orchestrator exiting with code=${code}"
|
|
||||||
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
stop_all_servers
|
|
||||||
fi
|
|
||||||
exit "$code"
|
|
||||||
}
|
|
||||||
trap on_exit EXIT
|
|
||||||
|
|
||||||
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
|
|
||||||
if ! wait_for_all_services "$PORTS"; then
|
|
||||||
log "error: not all healthy services found on ports ${PORTS}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
stop_all_servers
|
|
||||||
start_all_servers
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "===== BENCHMARK START ====="
|
|
||||||
|
|
||||||
for scenario in "${SCENARIOS[@]}"; do
|
|
||||||
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
|
||||||
# Fallback for legacy 3-field scenarios.
|
|
||||||
if [[ -z "${num_prompts:-}" ]]; then
|
|
||||||
num_prompts="$NUM_PROMPTS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
|
||||||
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
|
|
||||||
|
|
||||||
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
|
|
||||||
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports "$PORTS" \
|
|
||||||
--model "$SERVED_MODEL_NAME" \
|
|
||||||
--concurrency "$concurrency" \
|
|
||||||
--input-len "$input_len" \
|
|
||||||
--output-len "$output_len" \
|
|
||||||
--num-prompts "$num_prompts" \
|
|
||||||
--lb-strategy "$LB_STRATEGY" \
|
|
||||||
--health-check-interval 5 \
|
|
||||||
--health-max-failures 2 \
|
|
||||||
--health-recovery-interval 10 \
|
|
||||||
--request-max-retries 2 \
|
|
||||||
--output-file "$output_file" \
|
|
||||||
> "$detail_log" 2>&1 || {
|
|
||||||
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
|
||||||
# After failure, check service health before continuing.
|
|
||||||
check_and_report_services "$PORTS"
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log "finished scenario: output=${output_file}"
|
|
||||||
|
|
||||||
# Between scenarios, report service health so we can spot degradation early.
|
|
||||||
check_and_report_services "$PORTS" || true
|
|
||||||
done
|
|
||||||
|
|
||||||
log "===== BENCHMARK DONE ====="
|
|
||||||
|
|
||||||
log "parsing results"
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
|
|
||||||
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
|
|
||||||
}
|
|
||||||
|
|
||||||
log "all results saved to ${RESULT_ROOT}"
|
|
||||||
@ -1,107 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Start 2 independent vLLM TP=4 services on 8x H200.
|
|
||||||
# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7).
|
|
||||||
# This maximizes per-service throughput for higher-concurrency workloads.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
VENV="${VENV_SERVER}"
|
|
||||||
export PATH="$VENV/bin:$PATH"
|
|
||||||
VLLM="$VENV/bin/vllm"
|
|
||||||
|
|
||||||
TP=4
|
|
||||||
LOG_DIR="${SCRIPT_DIR}/logs"
|
|
||||||
mkdir -p "$LOG_DIR"
|
|
||||||
|
|
||||||
# 2 services, each on 4 GPUs.
|
|
||||||
# Port and GPU mapping.
|
|
||||||
SERVICES=(
|
|
||||||
"30005:0,1,2,3"
|
|
||||||
"30006:4,5,6,7"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Kill any existing vLLM processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
# Remove old PID files.
|
|
||||||
rm -f "${SCRIPT_DIR}"/*.pid
|
|
||||||
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
GPUS="${svc##*:}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
|
|
||||||
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
|
|
||||||
echo "Log: ${LOG}"
|
|
||||||
|
|
||||||
CUDA_VISIBLE_DEVICES="${GPUS}" \
|
|
||||||
nohup "$VLLM" serve "$MODEL_PATH" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--tensor-parallel-size "$TP" \
|
|
||||||
--block-size 256 \
|
|
||||||
--served-model-name "$SERVED_MODEL_NAME" \
|
|
||||||
--port "$PORT" \
|
|
||||||
> "$LOG" 2>&1 &
|
|
||||||
PID=$!
|
|
||||||
echo $PID > "$PID_FILE"
|
|
||||||
echo "PID: $PID"
|
|
||||||
done
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Waiting for all 2 services to become healthy..."
|
|
||||||
|
|
||||||
MAX_WAIT=240
|
|
||||||
all_healthy=0
|
|
||||||
for i in $(seq 1 $MAX_WAIT); do
|
|
||||||
all_healthy=1
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
|
|
||||||
all_healthy=0
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$all_healthy" -eq 1 ]]; then
|
|
||||||
echo "All 2 services are healthy!"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
echo " - http://127.0.0.1:${PORT}"
|
|
||||||
done
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if any process died early.
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
if [[ -f "$PID_FILE" ]]; then
|
|
||||||
PID="$(cat "$PID_FILE")"
|
|
||||||
if ! kill -0 "$PID" 2>/dev/null; then
|
|
||||||
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
tail -200 $LOG 2>/dev/null || true
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if (( i % 10 == 0 )); then
|
|
||||||
echo "Waiting... (${i}/${MAX_WAIT})"
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
echo "--- Last 50 lines of port ${PORT} ---"
|
|
||||||
tail -50 $LOG 2>/dev/null || true
|
|
||||||
done
|
|
||||||
exit 1
|
|
||||||
@ -228,16 +228,22 @@ class LoadBalancer:
|
|||||||
# Token helpers
|
# Token helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
import uuid
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
def make_prompt(token_len: int) -> str:
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
prefix = str(uuid.uuid4())
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
words = [
|
||||||
needed = max(token_len, 1)
|
"the", "quick", "brown", "fox",
|
||||||
tokens: list[str] = []
|
"jumps", "over", "lazy", "dog",
|
||||||
while len(tokens) < needed:
|
"benchmark", "performance", "latency"
|
||||||
|
]
|
||||||
|
|
||||||
|
tokens = [prefix]
|
||||||
|
|
||||||
|
while len(tokens) < token_len:
|
||||||
tokens.extend(words)
|
tokens.extend(words)
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
return " ".join(tokens[:token_len])
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@ -440,7 +446,36 @@ async def run_benchmark(
|
|||||||
return await send_request_with_retry(
|
return await send_request_with_retry(
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
session, host, lb, model, idx, prompt, output_len, temperature
|
||||||
)
|
)
|
||||||
|
print("Starting warmup...")
|
||||||
|
warmup_per_service = 10
|
||||||
|
warmup_requests = warmup_per_service * len(ports)
|
||||||
|
|
||||||
|
warmup_prompts = [
|
||||||
|
make_prompt(input_len)
|
||||||
|
for _ in range(warmup_requests)
|
||||||
|
]
|
||||||
|
|
||||||
|
warmup_tasks = [
|
||||||
|
send_request_with_retry(
|
||||||
|
session=session,
|
||||||
|
host=host,
|
||||||
|
lb=lb,
|
||||||
|
model=model,
|
||||||
|
request_id=-1,
|
||||||
|
prompt=p,
|
||||||
|
max_tokens=output_len,
|
||||||
|
temperature=temperature,
|
||||||
|
)
|
||||||
|
for p in warmup_prompts
|
||||||
|
]
|
||||||
|
|
||||||
|
await asyncio.gather(*warmup_tasks)
|
||||||
|
|
||||||
|
print("Warmup finished.")
|
||||||
|
|
||||||
|
# 给 CUDA / Scheduler 一点时间稳定
|
||||||
|
await asyncio.sleep(2)
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
await health_checker.stop()
|
await health_checker.stop()
|
||||||
|
|||||||
@ -32,51 +32,148 @@ LB_STRATEGY="round_robin"
|
|||||||
|
|
||||||
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
||||||
# For single-service TP=8 we cover standard concurrency gradients.
|
# For single-service TP=8 we cover standard concurrency gradients.
|
||||||
|
# concurrency input_len output_len num_prompts
|
||||||
SCENARIOS=(
|
SCENARIOS=(
|
||||||
"1 512 256 20"
|
"1 512 256 20"
|
||||||
"2 512 256 40"
|
"2 512 256 40"
|
||||||
"4 512 256 80"
|
"4 512 256 80"
|
||||||
"8 512 256 160"
|
"8 512 256 160"
|
||||||
"16 512 256 320"
|
"16 512 256 320"
|
||||||
"32 512 256 640"
|
"32 512 256 640"
|
||||||
"64 512 256 1280"
|
"64 512 256 1280"
|
||||||
"128 512 256 2560"
|
"128 512 256 2560"
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
"1 1024 128 20"
|
||||||
"32 2048 512 640"
|
"1 1024 256 20"
|
||||||
"128 2048 512 2560"
|
"1 1024 512 20"
|
||||||
"1 4096 1024 20"
|
"1 1024 1024 20"
|
||||||
"8 4096 1024 160"
|
"1 1024 2048 20"
|
||||||
"32 4096 1024 640"
|
"1 1024 4096 20"
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
"8 1024 128 160"
|
||||||
"8 8192 1024 160"
|
"8 1024 256 160"
|
||||||
"32 8192 1024 640"
|
"8 1024 512 160"
|
||||||
"128 8192 1024 2560"
|
"8 1024 1024 160"
|
||||||
"1 16384 1024 20"
|
"8 1024 2048 160"
|
||||||
"8 16384 1024 160"
|
"8 1024 4096 160"
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
"32 1024 128 640"
|
||||||
"1 32768 1024 20"
|
"32 1024 256 640"
|
||||||
"8 32768 1024 160"
|
"32 1024 512 640"
|
||||||
"32 32768 1024 640"
|
"32 1024 1024 640"
|
||||||
"128 32768 1024 2560"
|
"32 1024 2048 640"
|
||||||
"1 65536 1024 20"
|
"32 1024 4096 640"
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
"128 1024 128 2560"
|
||||||
"128 65536 1024 2560"
|
"128 1024 256 2560"
|
||||||
"1 131072 1024 20"
|
"128 1024 512 2560"
|
||||||
"8 131072 1024 160"
|
"128 1024 1024 2560"
|
||||||
"32 131072 1024 640"
|
"128 1024 2048 2560"
|
||||||
"128 131072 1024 2560"
|
"128 1024 4096 2560"
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
"1 2048 128 20"
|
||||||
"32 262144 1024 640"
|
"1 2048 256 20"
|
||||||
"128 262144 1024 2560"
|
"1 2048 512 20"
|
||||||
"1 524288 1024 20"
|
"1 2048 1024 20"
|
||||||
"8 524288 1024 160"
|
"1 2048 2048 20"
|
||||||
"32 524288 1024 640"
|
"1 2048 4096 20"
|
||||||
"128 524288 1024 2560"
|
|
||||||
|
"8 2048 128 160"
|
||||||
|
"8 2048 256 160"
|
||||||
|
"8 2048 512 160"
|
||||||
|
"8 2048 1024 160"
|
||||||
|
"8 2048 2048 160"
|
||||||
|
"8 2048 4096 160"
|
||||||
|
|
||||||
|
"32 2048 128 640"
|
||||||
|
"32 2048 256 640"
|
||||||
|
"32 2048 512 640"
|
||||||
|
"32 2048 1024 640"
|
||||||
|
"32 2048 2048 640"
|
||||||
|
"32 2048 4096 640"
|
||||||
|
|
||||||
|
"128 2048 128 2560"
|
||||||
|
"128 2048 256 2560"
|
||||||
|
"128 2048 512 2560"
|
||||||
|
"128 2048 1024 2560"
|
||||||
|
"128 2048 2048 2560"
|
||||||
|
"128 2048 4096 2560"
|
||||||
|
|
||||||
|
"1 4096 128 20"
|
||||||
|
"1 4096 256 20"
|
||||||
|
"1 4096 512 20"
|
||||||
|
"1 4096 1024 20"
|
||||||
|
"1 4096 2048 20"
|
||||||
|
"1 4096 4096 20"
|
||||||
|
|
||||||
|
"8 4096 128 160"
|
||||||
|
"8 4096 256 160"
|
||||||
|
"8 4096 512 160"
|
||||||
|
"8 4096 1024 160"
|
||||||
|
"8 4096 2048 160"
|
||||||
|
"8 4096 4096 160"
|
||||||
|
|
||||||
|
"32 4096 128 640"
|
||||||
|
"32 4096 256 640"
|
||||||
|
"32 4096 512 640"
|
||||||
|
"32 4096 1024 640"
|
||||||
|
"32 4096 2048 640"
|
||||||
|
"32 4096 4096 640"
|
||||||
|
|
||||||
|
"128 4096 128 2560"
|
||||||
|
"128 4096 256 2560"
|
||||||
|
"128 4096 512 2560"
|
||||||
|
"128 4096 1024 2560"
|
||||||
|
"128 4096 2048 2560"
|
||||||
|
"128 4096 4096 2560"
|
||||||
|
|
||||||
|
"1 16384 128 20"
|
||||||
|
"1 16384 256 20"
|
||||||
|
"1 16384 512 20"
|
||||||
|
"1 16384 1024 20"
|
||||||
|
"1 16384 2048 20"
|
||||||
|
|
||||||
|
"8 16384 128 160"
|
||||||
|
"8 16384 256 160"
|
||||||
|
"8 16384 512 160"
|
||||||
|
"8 16384 1024 160"
|
||||||
|
"8 16384 2048 160"
|
||||||
|
|
||||||
|
"32 16384 128 640"
|
||||||
|
"32 16384 256 640"
|
||||||
|
"32 16384 512 640"
|
||||||
|
"32 16384 1024 640"
|
||||||
|
"32 16384 2048 640"
|
||||||
|
|
||||||
|
"128 16384 128 2560"
|
||||||
|
"128 16384 256 2560"
|
||||||
|
"128 16384 512 2560"
|
||||||
|
"128 16384 1024 2560"
|
||||||
|
"128 16384 2048 2560"
|
||||||
|
|
||||||
|
|
||||||
|
"1 65536 128 20"
|
||||||
|
"1 65536 256 20"
|
||||||
|
"1 65536 512 20"
|
||||||
|
"1 65536 1024 20"
|
||||||
|
|
||||||
|
"8 65536 128 160"
|
||||||
|
"8 65536 256 160"
|
||||||
|
"8 65536 512 160"
|
||||||
|
"8 65536 1024 160"
|
||||||
|
"32 65536 1024 160"
|
||||||
|
|
||||||
|
"1 131072 128 20"
|
||||||
|
"1 131072 256 20"
|
||||||
|
"1 131072 512 20"
|
||||||
|
|
||||||
|
"8 131072 512 160"
|
||||||
|
|
||||||
|
"1 262144 256 20"
|
||||||
|
|
||||||
|
"8 262144 128 160"
|
||||||
|
|
||||||
|
"1 524288 128 20"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
# Server start script bundled with this experiment.
|
||||||
|
|||||||
@ -0,0 +1,129 @@
|
|||||||
|
# H200 vLLM TP=2 Custom Benchmark Report
|
||||||
|
|
||||||
|
- **Client**: `bench_client.py` (async OpenAI API, per-request timing)
|
||||||
|
- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)
|
||||||
|
- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756`
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |
|
||||||
|
|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||||
|
| c128_i1024_o1024 | 128 | 1024 | 1024 | 206.75 | 2560 | 0 | 12.38 | 12679.34 | 2588.49 | 15267.83 | 355.51 | 541.00 | 3399.25 | 46.35 | 50.58 | 52.54 | 9995.61 | 13713.98 | 15284.95 | ✅ |
|
||||||
|
| c128_i1024_o128 | 128 | 1024 | 128 | 133.01 | 2560 | 0 | 19.25 | 19708.67 | 2459.70 | 22168.36 | 1183.32 | 1479.29 | 3426.82 | 41.60 | 47.50 | 49.22 | 6458.71 | 7012.22 | 8759.52 | ✅ |
|
||||||
|
| c128_i1024_o2048 | 128 | 1024 | 2048 | 208.80 | 2560 | 0 | 12.26 | 12554.51 | 2547.16 | 15101.67 | 351.46 | 493.93 | 3429.21 | 47.17 | 52.36 | 59.29 | 10106.58 | 13980.86 | 15725.90 | ✅ |
|
||||||
|
| c128_i1024_o256 | 128 | 1024 | 256 | 205.11 | 2560 | 0 | 12.48 | 12780.63 | 2561.23 | 15341.86 | 351.22 | 456.94 | 3426.54 | 46.93 | 50.36 | 51.39 | 9935.31 | 12606.55 | 13028.92 | ✅ |
|
||||||
|
| c128_i1024_o4096 | 128 | 1024 | 4096 | 208.63 | 2560 | 0 | 12.27 | 12564.96 | 2562.18 | 15127.14 | 352.13 | 475.05 | 3401.06 | 46.82 | 50.35 | 51.59 | 10079.33 | 13938.76 | 15626.71 | ✅ |
|
||||||
|
| c128_i1024_o512 | 128 | 1024 | 512 | 210.21 | 2560 | 0 | 12.18 | 12470.38 | 2534.38 | 15004.76 | 355.66 | 519.46 | 3397.33 | 47.25 | 52.84 | 55.05 | 10138.54 | 13931.68 | 15398.92 | ✅ |
|
||||||
|
| c128_i16384_o128 | 128 | 16384 | 128 | 1359.07 | 2560 | 0 | 1.88 | 30861.66 | 240.91 | 31102.57 | 34308.65 | 34726.75 | 53850.14 | 257.93 | 261.13 | 261.25 | 67037.51 | 67882.99 | 86482.77 | ❌ |
|
||||||
|
| c128_i16384_o256 | 128 | 16384 | 256 | 1383.21 | 2560 | 0 | 1.85 | 30323.03 | 393.12 | 30716.14 | 13674.31 | 16619.00 | 53959.97 | 259.70 | 265.74 | 266.01 | 68522.28 | 82348.90 | 113143.30 | ❌ |
|
||||||
|
| c128_i16384_o512 | 128 | 16384 | 512 | 1394.58 | 2560 | 0 | 1.84 | 30075.73 | 449.14 | 30524.87 | 6929.19 | 14006.30 | 53929.18 | 256.42 | 266.39 | 266.68 | 68997.86 | 134484.18 | 142208.84 | ❌ |
|
||||||
|
| c128_i2048_o1024 | 128 | 2048 | 1024 | 274.18 | 2560 | 0 | 9.34 | 19121.84 | 2068.13 | 21189.97 | 541.55 | 764.67 | 6604.67 | 57.69 | 62.18 | 65.53 | 13253.98 | 21801.42 | 28048.56 | ⚠️ |
|
||||||
|
| c128_i2048_o128 | 128 | 2048 | 128 | 212.99 | 2560 | 0 | 12.02 | 24615.40 | 1530.02 | 26145.42 | 1293.09 | 1997.49 | 6703.31 | 72.33 | 80.38 | 81.78 | 10431.32 | 11370.95 | 15884.14 | ⚠️ |
|
||||||
|
| c128_i2048_o2048 | 128 | 2048 | 2048 | 272.06 | 2560 | 0 | 9.41 | 19270.69 | 2055.16 | 21325.85 | 542.66 | 696.46 | 6595.59 | 58.02 | 62.67 | 65.23 | 13143.51 | 21512.40 | 28583.36 | ⚠️ |
|
||||||
|
| c128_i2048_o256 | 128 | 2048 | 256 | 263.83 | 2560 | 0 | 9.70 | 19872.28 | 1986.56 | 21858.84 | 552.74 | 766.94 | 6667.42 | 60.36 | 64.80 | 67.61 | 12841.30 | 16455.00 | 19770.97 | ⚠️ |
|
||||||
|
| c128_i2048_o4096 | 128 | 2048 | 4096 | 274.18 | 2560 | 0 | 9.34 | 19121.82 | 2075.59 | 21197.41 | 538.85 | 772.70 | 6666.24 | 57.59 | 61.73 | 65.98 | 13271.96 | 22132.76 | 28679.85 | ⚠️ |
|
||||||
|
| c128_i2048_o512 | 128 | 2048 | 512 | 271.80 | 2560 | 0 | 9.42 | 19289.73 | 2074.26 | 21363.99 | 544.24 | 780.74 | 6725.67 | 57.75 | 61.62 | 65.40 | 13185.12 | 21822.20 | 28335.44 | ⚠️ |
|
||||||
|
| c128_i4096_o1024 | 128 | 4096 | 1024 | 424.83 | 2560 | 0 | 6.03 | 24682.53 | 1273.81 | 25956.34 | 1014.26 | 1488.91 | 13325.90 | 93.93 | 103.74 | 115.24 | 20697.67 | 35507.95 | 47170.14 | ⚠️ |
|
||||||
|
| c128_i4096_o128 | 128 | 4096 | 128 | 371.42 | 2560 | 0 | 6.89 | 28231.71 | 873.71 | 29105.42 | 1687.74 | 2056.57 | 13344.42 | 132.08 | 142.11 | 145.58 | 18310.58 | 19462.37 | 30169.28 | ⚠️ |
|
||||||
|
| c128_i4096_o2048 | 128 | 4096 | 2048 | 425.25 | 2560 | 0 | 6.02 | 24657.77 | 1288.03 | 25945.80 | 1019.46 | 1464.89 | 13332.86 | 93.13 | 101.52 | 111.69 | 20779.35 | 36358.76 | 50447.50 | ⚠️ |
|
||||||
|
| c128_i4096_o256 | 128 | 4096 | 256 | 413.80 | 2560 | 0 | 6.19 | 25340.14 | 1217.79 | 26557.93 | 1065.75 | 1554.31 | 13333.89 | 98.37 | 108.53 | 114.97 | 20311.65 | 26976.60 | 33054.57 | ⚠️ |
|
||||||
|
| c128_i4096_o4096 | 128 | 4096 | 4096 | 423.76 | 2560 | 0 | 6.04 | 24744.50 | 1282.56 | 26027.06 | 1040.84 | 1941.31 | 13328.94 | 93.34 | 101.81 | 111.13 | 20723.12 | 35255.54 | 48374.31 | ⚠️ |
|
||||||
|
| c128_i4096_o512 | 128 | 4096 | 512 | 423.70 | 2560 | 0 | 6.04 | 24748.17 | 1274.42 | 26022.58 | 1032.52 | 1562.56 | 13323.34 | 93.86 | 103.68 | 112.51 | 20669.59 | 35396.55 | 49034.77 | ⚠️ |
|
||||||
|
| c128_i512_o256 | 128 | 512 | 256 | 198.05 | 2560 | 0 | 12.93 | 6618.16 | 2673.83 | 9291.99 | 290.66 | 501.00 | 1944.87 | 45.16 | 52.68 | 57.19 | 9585.92 | 13135.34 | 14237.26 | ✅ |
|
||||||
|
| c16_i512_o256 | 16 | 512 | 256 | 70.00 | 320 | 0 | 4.57 | 2340.54 | 954.25 | 3294.79 | 155.99 | 299.17 | 421.01 | 14.77 | 16.43 | 17.44 | 3227.91 | 4110.18 | 4253.24 | ✅ |
|
||||||
|
| c1_i1024_o1024 | 1 | 1024 | 1024 | 41.00 | 20 | 0 | 0.49 | 499.49 | 102.02 | 601.51 | 113.27 | 116.29 | 137.79 | 6.73 | 6.74 | 6.74 | 1513.65 | 1949.27 | 1998.66 | ✅ |
|
||||||
|
| c1_i1024_o128 | 1 | 1024 | 128 | 26.62 | 20 | 0 | 0.75 | 769.29 | 96.16 | 865.45 | 112.10 | 114.09 | 127.55 | 6.69 | 6.70 | 6.70 | 961.87 | 964.11 | 977.09 | ✅ |
|
||||||
|
| c1_i1024_o2048 | 1 | 1024 | 2048 | 40.37 | 20 | 0 | 0.50 | 507.26 | 100.39 | 607.65 | 113.54 | 118.17 | 129.38 | 6.73 | 6.75 | 6.75 | 1470.10 | 1949.87 | 2093.40 | ✅ |
|
||||||
|
| c1_i1024_o256 | 1 | 1024 | 256 | 39.66 | 20 | 0 | 0.50 | 516.37 | 101.41 | 617.77 | 112.03 | 113.19 | 128.94 | 6.73 | 6.74 | 6.74 | 1457.72 | 1819.60 | 1819.96 | ✅ |
|
||||||
|
| c1_i1024_o4096 | 1 | 1024 | 4096 | 39.07 | 20 | 0 | 0.51 | 524.19 | 100.38 | 624.57 | 126.46 | 178.17 | 178.35 | 6.73 | 6.74 | 6.75 | 1439.54 | 1773.78 | 1831.83 | ✅ |
|
||||||
|
| c1_i1024_o512 | 1 | 1024 | 512 | 38.86 | 20 | 0 | 0.51 | 527.06 | 98.62 | 625.68 | 113.09 | 114.42 | 136.00 | 6.73 | 6.74 | 6.75 | 1396.05 | 1797.45 | 1820.39 | ✅ |
|
||||||
|
| c1_i16384_o1024 | 1 | 16384 | 1024 | 60.02 | 20 | 0 | 0.33 | 5459.87 | 74.28 | 5534.15 | 621.87 | 633.10 | 637.21 | 6.82 | 6.83 | 6.85 | 2136.32 | 2962.66 | 3211.92 | ✅ |
|
||||||
|
| c1_i16384_o128 | 1 | 16384 | 128 | 41.67 | 20 | 0 | 0.48 | 7862.85 | 60.88 | 7923.72 | 628.49 | 631.58 | 636.36 | 6.77 | 6.81 | 6.81 | 1480.09 | 1482.52 | 1482.98 | ✅ |
|
||||||
|
| c1_i16384_o2048 | 1 | 16384 | 2048 | 63.80 | 20 | 0 | 0.31 | 5136.27 | 80.76 | 5217.03 | 630.15 | 636.36 | 636.98 | 6.82 | 6.83 | 6.85 | 2381.45 | 4577.47 | 5209.66 | ✅ |
|
||||||
|
| c1_i16384_o256 | 1 | 16384 | 256 | 56.44 | 20 | 0 | 0.35 | 5805.62 | 73.65 | 5879.27 | 630.48 | 638.61 | 639.04 | 6.82 | 6.85 | 6.85 | 2040.24 | 2358.86 | 2362.47 | ✅ |
|
||||||
|
| c1_i16384_o512 | 1 | 16384 | 512 | 70.93 | 20 | 0 | 0.28 | 4619.51 | 85.87 | 4705.38 | 621.45 | 629.96 | 645.42 | 6.82 | 6.85 | 6.85 | 2692.18 | 4094.39 | 4094.98 | ✅ |
|
||||||
|
| c1_i2048_o1024 | 1 | 2048 | 1024 | 43.34 | 20 | 0 | 0.46 | 945.05 | 102.00 | 1047.06 | 123.15 | 127.95 | 148.69 | 6.80 | 6.81 | 6.81 | 1619.33 | 2410.40 | 2903.39 | ✅ |
|
||||||
|
| c1_i2048_o128 | 1 | 2048 | 128 | 27.02 | 20 | 0 | 0.74 | 1515.96 | 94.12 | 1610.08 | 122.85 | 126.65 | 147.47 | 6.77 | 6.77 | 6.82 | 976.42 | 983.68 | 1006.22 | ✅ |
|
||||||
|
| c1_i2048_o2048 | 1 | 2048 | 2048 | 43.35 | 20 | 0 | 0.46 | 944.86 | 101.45 | 1046.31 | 123.06 | 128.19 | 147.54 | 6.80 | 6.83 | 6.84 | 1611.88 | 2108.88 | 2496.01 | ✅ |
|
||||||
|
| c1_i2048_o256 | 1 | 2048 | 256 | 37.22 | 20 | 0 | 0.54 | 1100.59 | 96.68 | 1197.27 | 122.80 | 125.69 | 145.02 | 6.81 | 6.82 | 6.83 | 1340.69 | 1621.05 | 1775.55 | ✅ |
|
||||||
|
| c1_i2048_o4096 | 1 | 2048 | 4096 | 40.11 | 20 | 0 | 0.50 | 1021.08 | 101.61 | 1122.69 | 123.10 | 127.51 | 146.72 | 6.80 | 6.82 | 6.82 | 1502.66 | 2076.61 | 2137.65 | ✅ |
|
||||||
|
| c1_i2048_o512 | 1 | 2048 | 512 | 37.02 | 20 | 0 | 0.54 | 1106.44 | 96.52 | 1202.96 | 122.80 | 127.33 | 146.10 | 6.81 | 6.82 | 6.82 | 1331.89 | 1558.94 | 1578.90 | ✅ |
|
||||||
|
| c1_i4096_o1024 | 1 | 4096 | 1024 | 43.12 | 20 | 0 | 0.46 | 1899.75 | 99.83 | 1999.59 | 186.35 | 190.63 | 190.95 | 6.80 | 6.81 | 6.81 | 1644.18 | 2109.89 | 2311.93 | ✅ |
|
||||||
|
| c1_i4096_o128 | 1 | 4096 | 128 | 28.91 | 20 | 0 | 0.69 | 2833.86 | 88.56 | 2922.42 | 185.86 | 190.38 | 191.25 | 6.75 | 6.76 | 6.76 | 1043.43 | 1045.36 | 1047.58 | ✅ |
|
||||||
|
| c1_i4096_o2048 | 1 | 4096 | 2048 | 42.94 | 20 | 0 | 0.47 | 1907.88 | 96.51 | 2004.39 | 186.77 | 193.30 | 193.74 | 6.80 | 6.81 | 6.81 | 1589.25 | 2097.82 | 2452.99 | ✅ |
|
||||||
|
| c1_i4096_o256 | 1 | 4096 | 256 | 40.43 | 20 | 0 | 0.49 | 2026.29 | 93.65 | 2119.94 | 189.05 | 207.59 | 207.83 | 6.80 | 6.81 | 6.82 | 1469.51 | 1913.76 | 1913.87 | ✅ |
|
||||||
|
| c1_i4096_o4096 | 1 | 4096 | 4096 | 43.12 | 20 | 0 | 0.46 | 1899.82 | 96.29 | 1996.11 | 187.93 | 205.29 | 207.29 | 6.80 | 6.82 | 6.82 | 1593.63 | 2383.61 | 2729.94 | ✅ |
|
||||||
|
| c1_i4096_o512 | 1 | 4096 | 512 | 41.37 | 20 | 0 | 0.48 | 1980.16 | 89.19 | 2069.35 | 186.00 | 187.35 | 190.66 | 6.81 | 6.81 | 6.82 | 1434.80 | 2102.19 | 2396.89 | ✅ |
|
||||||
|
| c1_i512_o256 | 1 | 512 | 256 | 39.88 | 20 | 0 | 0.50 | 256.78 | 96.80 | 353.58 | 118.18 | 176.60 | 184.48 | 6.72 | 6.75 | 6.75 | 1408.44 | 1822.85 | 1871.30 | ✅ |
|
||||||
|
| c2_i512_o256 | 2 | 512 | 256 | 41.58 | 40 | 0 | 0.96 | 492.58 | 196.05 | 688.62 | 173.55 | 211.08 | 357.86 | 7.61 | 8.16 | 8.40 | 1706.70 | 2108.24 | 2165.31 | ✅ |
|
||||||
|
| c32_i1024_o1024 | 32 | 1024 | 1024 | 101.71 | 640 | 0 | 6.29 | 6443.13 | 1317.59 | 7760.73 | 197.93 | 311.27 | 984.94 | 22.03 | 23.78 | 24.21 | 4796.09 | 6445.23 | 7146.16 | ✅ |
|
||||||
|
| c32_i1024_o128 | 32 | 1024 | 128 | 53.75 | 640 | 0 | 11.91 | 12192.47 | 1523.87 | 13716.35 | 681.10 | 1046.63 | 1091.53 | 14.34 | 16.98 | 18.60 | 2502.18 | 2579.44 | 2641.32 | ✅ |
|
||||||
|
| c32_i1024_o2048 | 32 | 1024 | 2048 | 101.76 | 640 | 0 | 6.29 | 6440.44 | 1316.74 | 7757.18 | 204.19 | 335.85 | 1145.74 | 21.89 | 24.02 | 24.53 | 4767.74 | 6630.19 | 7109.51 | ✅ |
|
||||||
|
| c32_i1024_o256 | 32 | 1024 | 256 | 99.60 | 640 | 0 | 6.43 | 6579.98 | 1309.14 | 7889.12 | 200.93 | 331.50 | 921.88 | 22.15 | 24.50 | 25.32 | 4694.16 | 6056.40 | 6274.46 | ✅ |
|
||||||
|
| c32_i1024_o4096 | 32 | 1024 | 4096 | 103.10 | 640 | 0 | 6.21 | 6356.26 | 1309.63 | 7665.88 | 205.97 | 316.61 | 1157.52 | 22.04 | 24.05 | 24.79 | 4833.69 | 6458.13 | 7314.04 | ✅ |
|
||||||
|
| c32_i1024_o512 | 32 | 1024 | 512 | 103.06 | 640 | 0 | 6.21 | 6359.18 | 1304.32 | 7663.50 | 196.61 | 340.27 | 945.16 | 22.38 | 25.38 | 27.97 | 4872.21 | 6788.10 | 7689.83 | ✅ |
|
||||||
|
| c32_i16384_o1024 | 32 | 16384 | 1024 | 398.20 | 640 | 0 | 1.61 | 26333.15 | 402.85 | 26735.99 | 1772.26 | 3387.53 | 13518.46 | 70.46 | 85.49 | 101.84 | 19184.30 | 36508.49 | 54198.78 | ❌ |
|
||||||
|
| c32_i16384_o128 | 32 | 16384 | 128 | 359.20 | 640 | 0 | 1.78 | 29191.93 | 227.55 | 29419.48 | 3438.60 | 3983.92 | 13516.26 | 111.13 | 127.95 | 130.66 | 17518.50 | 19731.45 | 27923.40 | ❌ |
|
||||||
|
| c32_i16384_o2048 | 32 | 16384 | 2048 | 398.63 | 640 | 0 | 1.61 | 26304.81 | 403.52 | 26708.32 | 1715.04 | 2963.25 | 13514.42 | 70.47 | 86.65 | 95.98 | 19175.98 | 37536.35 | 50504.81 | ⚠️ |
|
||||||
|
| c32_i16384_o256 | 32 | 16384 | 256 | 383.92 | 640 | 0 | 1.67 | 27312.05 | 348.16 | 27660.21 | 1826.81 | 3388.69 | 13517.47 | 80.79 | 92.93 | 101.63 | 18646.62 | 24405.54 | 29695.09 | ❌ |
|
||||||
|
| c32_i16384_o512 | 32 | 16384 | 512 | 393.37 | 640 | 0 | 1.63 | 26656.12 | 383.78 | 27039.90 | 1747.04 | 3400.82 | 13513.48 | 73.78 | 90.30 | 96.96 | 19000.89 | 36338.55 | 41160.95 | ❌ |
|
||||||
|
| c32_i2048_o1024 | 32 | 2048 | 1024 | 112.12 | 640 | 0 | 5.71 | 11690.22 | 1270.45 | 12960.67 | 246.53 | 373.10 | 1896.26 | 22.21 | 24.25 | 25.25 | 5165.11 | 8407.83 | 11262.12 | ✅ |
|
||||||
|
| c32_i2048_o128 | 32 | 2048 | 128 | 73.91 | 640 | 0 | 8.66 | 17733.36 | 1105.41 | 18838.77 | 904.68 | 1464.61 | 1844.76 | 20.43 | 26.38 | 27.06 | 3492.80 | 3746.52 | 4348.11 | ✅ |
|
||||||
|
| c32_i2048_o2048 | 32 | 2048 | 2048 | 111.79 | 640 | 0 | 5.73 | 11725.08 | 1282.42 | 13007.50 | 246.50 | 442.18 | 1847.04 | 22.17 | 24.28 | 25.62 | 5185.73 | 8810.37 | 10858.07 | ✅ |
|
||||||
|
| c32_i2048_o256 | 32 | 2048 | 256 | 103.61 | 640 | 0 | 6.18 | 12650.11 | 1258.11 | 13908.22 | 240.98 | 367.88 | 1848.70 | 22.80 | 24.74 | 25.47 | 4859.84 | 6298.45 | 6687.99 | ✅ |
|
||||||
|
| c32_i2048_o4096 | 32 | 2048 | 4096 | 106.84 | 640 | 0 | 5.99 | 12268.59 | 1282.15 | 13550.74 | 250.26 | 439.47 | 1837.79 | 22.45 | 24.65 | 25.84 | 5036.07 | 8060.13 | 10159.85 | ✅ |
|
||||||
|
| c32_i2048_o512 | 32 | 2048 | 512 | 111.06 | 640 | 0 | 5.76 | 11802.41 | 1281.65 | 13084.06 | 242.26 | 437.24 | 1893.17 | 22.15 | 24.51 | 25.55 | 5136.64 | 8816.12 | 11310.11 | ✅ |
|
||||||
|
| c32_i4096_o1024 | 32 | 4096 | 1024 | 141.80 | 640 | 0 | 4.51 | 18486.39 | 947.32 | 19433.71 | 432.85 | 813.49 | 3483.06 | 30.04 | 33.92 | 36.32 | 6702.72 | 11605.60 | 14590.75 | ✅ |
|
||||||
|
| c32_i4096_o128 | 32 | 4096 | 128 | 114.34 | 640 | 0 | 5.60 | 22925.90 | 710.98 | 23636.88 | 1053.81 | 1909.50 | 3485.20 | 35.10 | 41.98 | 43.26 | 5478.89 | 6106.50 | 7458.03 | ✅ |
|
||||||
|
| c32_i4096_o2048 | 32 | 4096 | 2048 | 144.45 | 640 | 0 | 4.43 | 18147.38 | 939.67 | 19087.05 | 434.39 | 747.31 | 3483.20 | 29.88 | 33.53 | 35.93 | 6700.04 | 10714.15 | 14733.26 | ✅ |
|
||||||
|
| c32_i4096_o256 | 32 | 4096 | 256 | 137.28 | 640 | 0 | 4.66 | 19095.92 | 921.51 | 20017.42 | 441.00 | 731.75 | 3478.68 | 30.98 | 34.42 | 36.58 | 6528.50 | 8638.60 | 10034.50 | ✅ |
|
||||||
|
| c32_i4096_o4096 | 32 | 4096 | 4096 | 143.01 | 640 | 0 | 4.48 | 18330.54 | 961.50 | 19292.04 | 429.61 | 749.16 | 3469.34 | 29.60 | 33.17 | 35.01 | 6743.49 | 10500.86 | 14281.37 | ✅ |
|
||||||
|
| c32_i4096_o512 | 32 | 4096 | 512 | 143.19 | 640 | 0 | 4.47 | 18307.18 | 929.29 | 19236.47 | 430.74 | 691.68 | 3532.95 | 30.57 | 34.15 | 36.96 | 6735.35 | 10401.62 | 14892.06 | ✅ |
|
||||||
|
| c32_i512_o256 | 32 | 512 | 256 | 98.73 | 640 | 0 | 6.48 | 3319.01 | 1330.82 | 4649.83 | 186.98 | 378.34 | 678.28 | 21.82 | 23.82 | 24.54 | 4649.84 | 6039.51 | 6163.96 | ✅ |
|
||||||
|
| c4_i512_o256 | 4 | 512 | 256 | 46.00 | 80 | 0 | 1.74 | 890.42 | 360.47 | 1250.89 | 164.64 | 347.47 | 374.55 | 8.92 | 10.51 | 11.09 | 1997.44 | 2523.64 | 2675.43 | ✅ |
|
||||||
|
| c64_i512_o256 | 64 | 512 | 256 | 140.30 | 1280 | 0 | 9.12 | 4671.07 | 1901.53 | 6572.60 | 229.06 | 453.78 | 1147.81 | 31.26 | 35.35 | 36.92 | 6712.71 | 8741.37 | 9237.35 | ✅ |
|
||||||
|
| c8_i1024_o1024 | 8 | 1024 | 1024 | 54.01 | 160 | 0 | 2.96 | 3033.30 | 621.77 | 3655.07 | 144.65 | 218.00 | 394.00 | 10.80 | 11.82 | 12.24 | 2395.48 | 3214.61 | 3506.80 | ✅ |
|
||||||
|
| c8_i1024_o128 | 8 | 1024 | 128 | 30.89 | 160 | 0 | 5.18 | 5304.66 | 662.82 | 5967.48 | 307.64 | 345.56 | 389.57 | 8.30 | 9.77 | 9.80 | 1360.75 | 1376.96 | 1412.43 | ✅ |
|
||||||
|
| c8_i1024_o2048 | 8 | 1024 | 2048 | 53.44 | 160 | 0 | 2.99 | 3065.61 | 627.08 | 3692.69 | 147.14 | 284.31 | 375.40 | 10.76 | 11.60 | 12.31 | 2391.87 | 3190.58 | 3558.55 | ✅ |
|
||||||
|
| c8_i1024_o256 | 8 | 1024 | 256 | 52.32 | 160 | 0 | 3.06 | 3131.63 | 620.55 | 3752.18 | 144.06 | 216.85 | 383.28 | 10.89 | 11.75 | 12.29 | 2345.43 | 2965.11 | 3042.16 | ✅ |
|
||||||
|
| c8_i1024_o4096 | 8 | 1024 | 4096 | 55.32 | 160 | 0 | 2.89 | 2961.58 | 621.49 | 3583.07 | 144.43 | 220.03 | 380.43 | 10.73 | 11.56 | 12.09 | 2438.42 | 3382.06 | 3756.38 | ✅ |
|
||||||
|
| c8_i1024_o512 | 8 | 1024 | 512 | 54.03 | 160 | 0 | 2.96 | 3032.14 | 632.21 | 3664.35 | 144.40 | 216.10 | 375.86 | 10.74 | 11.48 | 11.76 | 2425.82 | 3208.35 | 3616.77 | ✅ |
|
||||||
|
| c8_i16384_o1024 | 8 | 16384 | 1024 | 138.97 | 160 | 0 | 1.15 | 18863.47 | 287.59 | 19151.06 | 967.72 | 1600.40 | 3663.33 | 21.47 | 27.32 | 29.59 | 6298.76 | 12516.84 | 14588.60 | ✅ |
|
||||||
|
| c8_i16384_o128 | 8 | 16384 | 128 | 111.93 | 160 | 0 | 1.43 | 23420.44 | 181.33 | 23601.76 | 2556.64 | 4119.31 | 4136.10 | 20.74 | 33.97 | 34.00 | 5167.54 | 5417.75 | 5933.67 | ⚠️ |
|
||||||
|
| c8_i16384_o2048 | 8 | 16384 | 2048 | 136.26 | 160 | 0 | 1.17 | 19238.95 | 275.01 | 19513.96 | 961.94 | 1572.87 | 3673.56 | 22.52 | 28.43 | 30.68 | 6163.97 | 11684.40 | 14272.99 | ✅ |
|
||||||
|
| c8_i16384_o256 | 8 | 16384 | 256 | 127.82 | 160 | 0 | 1.25 | 20509.06 | 253.69 | 20762.74 | 1027.09 | 1938.52 | 3668.72 | 24.19 | 29.35 | 30.47 | 5890.64 | 7584.77 | 7970.70 | ✅ |
|
||||||
|
| c8_i16384_o512 | 8 | 16384 | 512 | 136.61 | 160 | 0 | 1.17 | 19188.83 | 286.09 | 19474.92 | 969.45 | 1904.35 | 3667.98 | 21.65 | 26.89 | 29.25 | 6250.80 | 10966.73 | 12572.72 | ✅ |
|
||||||
|
| c8_i2048_o1024 | 8 | 2048 | 1024 | 59.22 | 160 | 0 | 2.70 | 5533.53 | 606.82 | 6140.34 | 155.89 | 244.40 | 621.82 | 10.83 | 11.86 | 12.11 | 2572.18 | 4486.64 | 5823.30 | ✅ |
|
||||||
|
| c8_i2048_o128 | 8 | 2048 | 128 | 36.79 | 160 | 0 | 4.35 | 8907.76 | 554.37 | 9462.13 | 337.85 | 507.02 | 613.47 | 10.27 | 12.09 | 12.51 | 1636.65 | 1709.00 | 1726.45 | ✅ |
|
||||||
|
| c8_i2048_o2048 | 8 | 2048 | 2048 | 57.55 | 160 | 0 | 2.78 | 5693.86 | 626.97 | 6320.83 | 156.06 | 275.62 | 603.04 | 10.76 | 11.60 | 11.99 | 2571.87 | 4101.50 | 5201.48 | ✅ |
|
||||||
|
| c8_i2048_o256 | 8 | 2048 | 256 | 52.19 | 160 | 0 | 3.07 | 6279.15 | 604.00 | 6883.15 | 158.73 | 268.03 | 611.54 | 11.13 | 12.15 | 12.45 | 2334.26 | 3004.26 | 3124.59 | ✅ |
|
||||||
|
| c8_i2048_o4096 | 8 | 2048 | 4096 | 59.22 | 160 | 0 | 2.70 | 5533.71 | 614.83 | 6148.54 | 164.46 | 304.53 | 685.46 | 10.84 | 11.77 | 11.98 | 2624.68 | 4094.22 | 5661.19 | ✅ |
|
||||||
|
| c8_i2048_o512 | 8 | 2048 | 512 | 57.76 | 160 | 0 | 2.77 | 5672.85 | 605.80 | 6278.66 | 154.18 | 248.11 | 617.88 | 10.86 | 11.81 | 12.00 | 2521.32 | 3808.31 | 4671.03 | ✅ |
|
||||||
|
| c8_i4096_o1024 | 8 | 4096 | 1024 | 64.92 | 160 | 0 | 2.46 | 10094.27 | 518.13 | 10612.40 | 256.33 | 442.48 | 1016.15 | 12.36 | 13.77 | 14.46 | 2846.88 | 4602.44 | 5170.71 | ✅ |
|
||||||
|
| c8_i4096_o128 | 8 | 4096 | 128 | 46.47 | 160 | 0 | 3.44 | 14102.60 | 440.04 | 14542.64 | 707.03 | 1047.28 | 1071.50 | 10.92 | 15.03 | 15.06 | 2091.88 | 2145.56 | 2225.52 | ✅ |
|
||||||
|
| c8_i4096_o2048 | 8 | 4096 | 2048 | 66.29 | 160 | 0 | 2.41 | 9886.22 | 513.33 | 10399.56 | 253.25 | 423.32 | 1022.59 | 12.39 | 14.05 | 14.66 | 2865.36 | 4853.71 | 6028.24 | ✅ |
|
||||||
|
| c8_i4096_o256 | 8 | 4096 | 256 | 62.26 | 160 | 0 | 2.57 | 10526.27 | 516.50 | 11042.77 | 256.86 | 457.13 | 1013.55 | 12.63 | 14.05 | 14.57 | 2789.27 | 3625.36 | 3833.62 | ✅ |
|
||||||
|
| c8_i4096_o4096 | 8 | 4096 | 4096 | 64.28 | 160 | 0 | 2.49 | 10195.14 | 543.84 | 10738.99 | 248.19 | 462.57 | 1014.75 | 12.27 | 13.68 | 14.32 | 2921.13 | 5570.16 | 8112.52 | ✅ |
|
||||||
|
| c8_i4096_o512 | 8 | 4096 | 512 | 65.25 | 160 | 0 | 2.45 | 10043.48 | 538.74 | 10582.22 | 247.69 | 442.69 | 1015.54 | 12.25 | 13.71 | 14.06 | 2931.06 | 5124.89 | 6513.32 | ✅ |
|
||||||
|
| c8_i512_o256 | 8 | 512 | 256 | 52.73 | 160 | 0 | 3.03 | 1553.68 | 629.48 | 2183.16 | 135.86 | 271.88 | 289.07 | 10.80 | 11.89 | 12.07 | 2369.82 | 3039.64 | 3126.47 | ✅ |
|
||||||
|
| c128_i16384_o1024 | 128 | 16384 | 1024 | 1401.24 | 2560 | 0 | 1.83 | 29932.73 | 453.17 | 30385.90 | 6827.03 | 13719.55 | 53845.73 | 254.25 | 266.59 | 267.01 | 69085.30 | 135447.23 | 185844.63 | ❌ |
|
||||||
|
| c128_i16384_o2048 | 128 | 16384 | 2048 | 1400.62 | 2560 | 0 | 1.83 | 29945.97 | 468.45 | 30414.41 | 5874.33 | 13475.14 | 53934.11 | 249.38 | 266.26 | 266.71 | 69277.40 | 141505.20 | 189264.63 | ❌ |
|
||||||
|
| c1_i131072_o128 | 1 | 131072 | 128 | 182.19 | 20 | 0 | 0.11 | 14388.29 | 12.72 | 14401.01 | 5392.41 | 5419.10 | 5420.54 | 6.96 | 6.97 | 6.98 | 6192.49 | 6221.42 | 6221.43 | ⚠️ |
|
||||||
|
| c1_i131072_o256 | 1 | 131072 | 256 | 195.04 | 20 | 0 | 0.10 | 13440.21 | 19.01 | 13459.22 | 5397.54 | 5431.66 | 5440.22 | 7.00 | 7.03 | 7.03 | 6686.72 | 7103.29 | 7119.59 | ⚠️ |
|
||||||
|
| c1_i131072_o512 | 1 | 131072 | 512 | 201.48 | 20 | 0 | 0.10 | 13010.69 | 20.19 | 13030.87 | 5500.61 | 5555.72 | 5562.25 | 7.00 | 7.02 | 7.03 | 6917.34 | 7364.34 | 8109.96 | ⚠️ |
|
||||||
|
| c1_i262144_o256 | 1 | 262144 | 256 | 426.96 | 20 | 0 | 0.05 | 12279.65 | 8.19 | 12287.84 | 13218.70 | 13504.54 | 13586.50 | 7.22 | 7.32 | 7.36 | 14470.75 | 14834.79 | 14835.15 | ⚠️ |
|
||||||
|
| c1_i524288_o128 | 1 | 524288 | 128 | 1098.28 | 20 | 0 | 0.02 | 9547.46 | 1.62 | 9549.08 | 36239.34 | 36595.44 | 37102.25 | 9.37 | 9.80 | 37.00 | 36907.34 | 37308.40 | 37807.76 | ⚠️ |
|
||||||
|
| c1_i65536_o1024 | 1 | 65536 | 1024 | 114.83 | 20 | 0 | 0.17 | 11414.14 | 39.37 | 11453.51 | 2462.78 | 2475.26 | 2495.15 | 6.92 | 6.93 | 6.93 | 4020.21 | 5428.03 | 7058.64 | ✅ |
|
||||||
|
| c1_i65536_o128 | 1 | 65536 | 128 | 96.37 | 20 | 0 | 0.21 | 13600.71 | 24.89 | 13625.61 | 2497.10 | 2528.43 | 2558.93 | 6.87 | 6.87 | 6.88 | 3313.69 | 3344.68 | 3373.65 | ✅ |
|
||||||
|
| c1_i65536_o256 | 1 | 65536 | 256 | 105.96 | 20 | 0 | 0.19 | 12370.49 | 32.66 | 12403.15 | 2460.46 | 2472.33 | 2483.17 | 6.92 | 6.93 | 6.93 | 3650.21 | 4159.14 | 4159.75 | ✅ |
|
||||||
|
| c1_i65536_o512 | 1 | 65536 | 512 | 108.81 | 20 | 0 | 0.18 | 12045.94 | 34.57 | 12080.52 | 2461.82 | 2468.77 | 2498.35 | 6.92 | 6.93 | 6.93 | 3756.97 | 4315.50 | 4593.51 | ✅ |
|
||||||
|
| c32_i65536_o1024 | 32 | 65536 | 1024 | 395.45 | 160 | 0 | 0.40 | 26515.91 | 89.22 | 26605.13 | 18530.86 | 54451.46 | 68783.28 | 252.00 | 283.71 | 284.41 | 72731.85 | 117936.92 | 151338.20 | ❌ |
|
||||||
|
| c8_i131072_o512 | 8 | 131072 | 512 | 887.91 | 160 | 0 | 0.18 | 23619.01 | 33.13 | 23652.14 | 12820.61 | 25478.95 | 32945.62 | 156.62 | 223.19 | 294.51 | 41479.53 | 66809.84 | 79279.57 | ❌ |
|
||||||
|
| c8_i262144_o128 | 8 | 262144 | 128 | 2121.21 | 160 | 0 | 0.08 | 19773.16 | 9.48 | 19782.64 | 51391.65 | 61587.85 | 80661.75 | 382.04 | 388.68 | 389.24 | 99110.05 | 110875.08 | 129961.05 | ❌ |
|
||||||
|
| c8_i65536_o1024 | 8 | 65536 | 1024 | 422.41 | 160 | 0 | 0.38 | 24823.70 | 76.89 | 24900.59 | 4665.85 | 8696.15 | 14818.98 | 73.86 | 99.00 | 113.60 | 19699.15 | 35998.21 | 56237.93 | ❌ |
|
||||||
|
| c8_i65536_o128 | 8 | 65536 | 128 | 402.35 | 160 | 0 | 0.40 | 26061.00 | 46.48 | 26107.48 | 8659.91 | 13193.45 | 15765.71 | 87.87 | 143.78 | 166.11 | 18805.63 | 21397.73 | 22483.11 | ❌ |
|
||||||
|
| c8_i65536_o256 | 8 | 65536 | 256 | 416.75 | 160 | 0 | 0.38 | 25160.80 | 67.25 | 25228.06 | 5113.03 | 10698.95 | 14785.06 | 82.53 | 113.13 | 132.88 | 19445.90 | 27248.32 | 32918.89 | ❌ |
|
||||||
|
| c8_i65536_o512 | 8 | 65536 | 512 | 422.66 | 160 | 0 | 0.38 | 24809.03 | 76.23 | 24885.26 | 4532.43 | 9015.11 | 14789.72 | 75.30 | 108.22 | 126.16 | 19689.27 | 34072.42 | 42942.27 | ❌ |
|
||||||
|
|
||||||
|
SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -71,7 +71,6 @@ data["config"] = {
|
|||||||
"gpu_groups": "0,1,2,3,4,5,6,7",
|
"gpu_groups": "0,1,2,3,4,5,6,7",
|
||||||
"kv_cache_dtype": "fp8",
|
"kv_cache_dtype": "fp8",
|
||||||
"block_size": 256,
|
"block_size": 256,
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
"client": "bench_client.py",
|
||||||
"lb_strategy": "round_robin",
|
"lb_strategy": "round_robin",
|
||||||
"scenarios": [
|
"scenarios": [
|
||||||
|
|||||||
@ -1,103 +0,0 @@
|
|||||||
# dsv4_h200_vllm_tp8_custom_bench_no_fp8
|
|
||||||
|
|
||||||
> **Note:** This variant does NOT use `--kv-cache-dtype fp8`.
|
|
||||||
|
|
||||||
## 目的
|
|
||||||
|
|
||||||
当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU：
|
|
||||||
|
|
||||||
| 服务 | 端口 | GPU 组 |
|
|
||||||
|------|------|--------|
|
|
||||||
| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 |
|
|
||||||
|
|
||||||
TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。
|
|
||||||
|
|
||||||
## 与多服务方案的区别
|
|
||||||
|
|
||||||
| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) |
|
|
||||||
|------|---------------|---------------|---------------|
|
|
||||||
| 服务数量 | 4 | 2 | 1 |
|
|
||||||
| 每服务 GPU | 2 | 4 | 8 |
|
|
||||||
| 单服务吞吐 | 较低 | 中等 | 最高 |
|
|
||||||
| 集群总吞吐 | 相近 | 相近 | 相近 |
|
|
||||||
| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch |
|
|
||||||
|
|
||||||
## 文件说明
|
|
||||||
|
|
||||||
| 文件 | 说明 |
|
|
||||||
|------|------|
|
|
||||||
| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） |
|
|
||||||
| `config.env` | 实验配置（单服务、端口 30005） |
|
|
||||||
| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 |
|
|
||||||
| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） |
|
|
||||||
| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` |
|
|
||||||
|
|
||||||
## 快速运行
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 完整流程（自动启动服务、压测、生成报告）
|
|
||||||
bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 跳过服务管理（已有服务在运行）
|
|
||||||
SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh
|
|
||||||
|
|
||||||
# 单独运行压测客户端（单服务）
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \
|
|
||||||
--output-file /tmp/test.jsonl
|
|
||||||
|
|
||||||
# 解析已有结果
|
|
||||||
python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \
|
|
||||||
experiments/dsv4_h200_vllm_tp8_custom_bench/results/<RUN_ID>
|
|
||||||
```
|
|
||||||
|
|
||||||
## 场景设计
|
|
||||||
|
|
||||||
本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合：
|
|
||||||
|
|
||||||
| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 |
|
|
||||||
|------|----------|----------|--------|------|
|
|
||||||
| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 |
|
|
||||||
| 2 | 512 | 256 | 40 | 低并发 |
|
|
||||||
| 4 | 512 | 256 | 80 | 中低并发 |
|
|
||||||
| 8 | 512 | 256 | 160 | 中等并发 |
|
|
||||||
| 16 | 512 | 256 | 320 | 中高并发 |
|
|
||||||
| 32 | 512 | 256 | 640 | 高并发 |
|
|
||||||
| 64 | 512 | 256 | 1280 | 很高并发 |
|
|
||||||
| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 |
|
|
||||||
| 1 | 2048 | 512 | 20 | 长输入基线 |
|
|
||||||
| 8 | 2048 | 512 | 160 | 长输入中并发 |
|
|
||||||
| 32 | 2048 | 512 | 640 | 长输入高并发 |
|
|
||||||
| 128 | 2048 | 512 | 2560 | 长输入极限并发 |
|
|
||||||
| 1 | 4096 | 1024 | 20 | 超长输入基线 |
|
|
||||||
| 8 | 4096 | 1024 | 160 | 超长输入中并发 |
|
|
||||||
| 32 | 4096 | 1024 | 640 | 超长输入高并发 |
|
|
||||||
| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 |
|
|
||||||
|
|
||||||
## 输出格式
|
|
||||||
|
|
||||||
同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。
|
|
||||||
|
|
||||||
## 单独启动/停止服务
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 启动服务
|
|
||||||
bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh
|
|
||||||
|
|
||||||
# 停止（run_bench.sh 会自动停止，但也可以手动）
|
|
||||||
port=30005
|
|
||||||
pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid"
|
|
||||||
[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true
|
|
||||||
rm -f "$pid_file"
|
|
||||||
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
|
|
||||||
```
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
|
|
||||||
- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。
|
|
||||||
- 模型文件路径：`/data/models/DeepSeek-V4-Flash`
|
|
||||||
- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。
|
|
||||||
- 健康检查依赖 `/health` 端点（vLLM 默认启用）。
|
|
||||||
- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。
|
|
||||||
@ -1,617 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover.
|
|
||||||
|
|
||||||
Sends concurrent requests to multiple inference servers and records per-request
|
|
||||||
latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing.
|
|
||||||
|
|
||||||
Designed for TP=2 on 8-GPU setups where 4 independent services run on
|
|
||||||
(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via
|
|
||||||
round-robin or random load balancing. Unhealthy services are automatically
|
|
||||||
skipped with periodic retry.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
# Single service (backward compatible)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 --port 30005 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
# Multi-service (4 services on ports 30005-30008)
|
|
||||||
python3 bench_client.py \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--ports 30005,30006,30007,30008 \
|
|
||||||
--model deepseek-v4-flash \
|
|
||||||
--concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \
|
|
||||||
--lb-strategy round_robin \
|
|
||||||
--output-file results.jsonl
|
|
||||||
|
|
||||||
The output JSONL contains one object per request with raw timing data.
|
|
||||||
A final summary line ("completed" field) aggregates all requests.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
import random
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import urllib.request
|
|
||||||
from dataclasses import asdict, dataclass, field
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
# Try aiohttp first; fall back to urllib for stdlib-only environments.
|
|
||||||
try:
|
|
||||||
import aiohttp
|
|
||||||
HAS_AIOHTTP = True
|
|
||||||
except ImportError:
|
|
||||||
HAS_AIOHTTP = False
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Data structures
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class RequestResult:
|
|
||||||
request_id: int
|
|
||||||
input_tokens: int = 0
|
|
||||||
output_tokens: int = 0
|
|
||||||
first_token_time_ms: float = 0.0 # TTFT
|
|
||||||
e2e_latency_ms: float = 0.0 # total wall time
|
|
||||||
inter_token_latencies: list[float] = field(default_factory=list)
|
|
||||||
success: bool = False
|
|
||||||
error: str = ""
|
|
||||||
target_port: int = 0 # which service handled this request
|
|
||||||
retry_count: int = 0 # how many times this request was retried
|
|
||||||
|
|
||||||
@property
|
|
||||||
def tpot_ms(self) -> float:
|
|
||||||
"""Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token."""
|
|
||||||
if self.output_tokens <= 1:
|
|
||||||
return 0.0
|
|
||||||
return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def mean_itl_ms(self) -> float:
|
|
||||||
if not self.inter_token_latencies:
|
|
||||||
return 0.0
|
|
||||||
return sum(self.inter_token_latencies) / len(self.inter_token_latencies)
|
|
||||||
|
|
||||||
def to_dict(self) -> dict[str, Any]:
|
|
||||||
d = asdict(self)
|
|
||||||
d["tpot_ms"] = self.tpot_ms
|
|
||||||
d["mean_itl_ms"] = self.mean_itl_ms
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Health checker
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class HealthChecker:
|
|
||||||
"""Periodically check service health and maintain a healthy-port list."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
check_interval: float = 5.0,
|
|
||||||
max_failures: int = 2,
|
|
||||||
recovery_interval: float = 10.0,
|
|
||||||
):
|
|
||||||
self.host = host
|
|
||||||
self.all_ports = ports
|
|
||||||
self.check_interval = check_interval
|
|
||||||
self.max_failures = max_failures
|
|
||||||
self.recovery_interval = recovery_interval
|
|
||||||
|
|
||||||
# port -> consecutive failure count
|
|
||||||
self._failures: dict[int, int] = {p: 0 for p in ports}
|
|
||||||
# port -> last healthy timestamp
|
|
||||||
self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports}
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
self._task: asyncio.Task | None = None
|
|
||||||
|
|
||||||
async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool:
|
|
||||||
url = f"http://{self.host}:{port}/health"
|
|
||||||
try:
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
else:
|
|
||||||
req = urllib.request.Request(url, method="GET")
|
|
||||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
||||||
return resp.status == 200
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def _check_all(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
checks = [self._check_one(p, session) for p in self.all_ports]
|
|
||||||
results = await asyncio.gather(*checks, return_exceptions=True)
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
for port, ok in zip(self.all_ports, results):
|
|
||||||
if isinstance(ok, Exception):
|
|
||||||
ok = False
|
|
||||||
if ok:
|
|
||||||
self._failures[port] = 0
|
|
||||||
self._last_healthy[port] = now
|
|
||||||
else:
|
|
||||||
self._failures[port] += 1
|
|
||||||
|
|
||||||
async def start(self, session: aiohttp.ClientSession | None) -> None:
|
|
||||||
async def _loop() -> None:
|
|
||||||
while True:
|
|
||||||
await asyncio.sleep(self.check_interval)
|
|
||||||
await self._check_all(session)
|
|
||||||
self._task = asyncio.create_task(_loop())
|
|
||||||
|
|
||||||
async def stop(self) -> None:
|
|
||||||
if self._task is not None:
|
|
||||||
self._task.cancel()
|
|
||||||
try:
|
|
||||||
await self._task
|
|
||||||
except asyncio.CancelledError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@property
|
|
||||||
async def healthy_ports(self) -> list[int]:
|
|
||||||
now = time.monotonic()
|
|
||||||
async with self._lock:
|
|
||||||
healthy = []
|
|
||||||
for p in self.all_ports:
|
|
||||||
if self._failures[p] < self.max_failures:
|
|
||||||
healthy.append(p)
|
|
||||||
elif now - self._last_healthy[p] > self.recovery_interval:
|
|
||||||
# Give it another chance after recovery_interval.
|
|
||||||
self._failures[p] = max(0, self._failures[p] - 1)
|
|
||||||
healthy.append(p)
|
|
||||||
return healthy
|
|
||||||
|
|
||||||
async def is_healthy(self, port: int) -> bool:
|
|
||||||
ports = await self.healthy_ports
|
|
||||||
return port in ports
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Load balancer with failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
class LoadBalancer:
|
|
||||||
"""Distribute requests across healthy backend ports with auto-failover."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
ports: list[int],
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
strategy: str = "round_robin",
|
|
||||||
):
|
|
||||||
self.all_ports = ports
|
|
||||||
self.health_checker = health_checker
|
|
||||||
self.strategy = strategy
|
|
||||||
self._idx = 0
|
|
||||||
self._lock = asyncio.Lock()
|
|
||||||
|
|
||||||
def _next_round_robin(self, healthy: list[int]) -> int:
|
|
||||||
if not healthy:
|
|
||||||
return self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
for _ in range(len(self.all_ports)):
|
|
||||||
port = self.all_ports[self._idx % len(self.all_ports)]
|
|
||||||
self._idx += 1
|
|
||||||
if port in healthy:
|
|
||||||
return port
|
|
||||||
# Fallback: all unhealthy, pick first healthy anyway.
|
|
||||||
return healthy[0] if healthy else self.all_ports[0]
|
|
||||||
|
|
||||||
def _next_random(self, healthy: list[int]) -> int:
|
|
||||||
if healthy:
|
|
||||||
return random.choice(healthy)
|
|
||||||
return random.choice(self.all_ports)
|
|
||||||
|
|
||||||
async def next_port(self) -> int:
|
|
||||||
healthy = await self.health_checker.healthy_ports
|
|
||||||
async with self._lock:
|
|
||||||
if self.strategy == "round_robin":
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
elif self.strategy in ("random", "least_conn"):
|
|
||||||
return self._next_random(healthy)
|
|
||||||
else:
|
|
||||||
return self._next_round_robin(healthy)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def port_count(self) -> int:
|
|
||||||
return len(self.all_ports)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Token helpers
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def make_prompt(token_len: int, vocab_size: int = 32000) -> str:
|
|
||||||
"""Generate a random-ish prompt of approximately `token_len` tokens."""
|
|
||||||
words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy",
|
|
||||||
"dog", "hello", "world", "deep", "seek", "model", "test",
|
|
||||||
"benchmark", "performance", "latency", "throughput", "token"]
|
|
||||||
needed = max(token_len, 1)
|
|
||||||
tokens: list[str] = []
|
|
||||||
while len(tokens) < needed:
|
|
||||||
tokens.extend(words)
|
|
||||||
return " ".join(tokens[:needed])
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# aiohttp-based async request (preferred)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_aiohttp(
|
|
||||||
session: aiohttp.ClientSession,
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": True,
|
|
||||||
}
|
|
||||||
headers = {"Content-Type": "application/json"}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
first_token_received = False
|
|
||||||
last_token_time = 0.0
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with session.post(url, json=payload, headers=headers) as resp:
|
|
||||||
if resp.status != 200:
|
|
||||||
text = await resp.text()
|
|
||||||
result.error = f"HTTP {resp.status}: {text[:200]}"
|
|
||||||
return result
|
|
||||||
|
|
||||||
async for line in resp.content:
|
|
||||||
line = line.decode("utf-8").strip()
|
|
||||||
if not line or not line.startswith("data: "):
|
|
||||||
continue
|
|
||||||
data_str = line[len("data: "):]
|
|
||||||
if data_str == "[DONE]":
|
|
||||||
break
|
|
||||||
try:
|
|
||||||
chunk = json.loads(data_str)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
choices = chunk.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
continue
|
|
||||||
|
|
||||||
delta = choices[0].get("delta", {})
|
|
||||||
content = delta.get("content", "")
|
|
||||||
if content:
|
|
||||||
now = time.perf_counter()
|
|
||||||
if not first_token_received:
|
|
||||||
result.first_token_time_ms = (now - start_time) * 1000
|
|
||||||
first_token_received = True
|
|
||||||
else:
|
|
||||||
result.inter_token_latencies.append((now - last_token_time) * 1000)
|
|
||||||
last_token_time = now
|
|
||||||
result.output_tokens += 1
|
|
||||||
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.input_tokens = len(prompt.split()) # approximate
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
result.error = "timeout"
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# urllib-based synchronous request (fallback, no aiohttp)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def send_request_urllib(
|
|
||||||
host: str,
|
|
||||||
port: int,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Blocking fallback using stdlib urllib. Used when aiohttp is absent."""
|
|
||||||
url = f"http://{host}:{port}/v1/chat/completions"
|
|
||||||
payload = {
|
|
||||||
"model": model,
|
|
||||||
"messages": [{"role": "user", "content": prompt}],
|
|
||||||
"max_tokens": max_tokens,
|
|
||||||
"temperature": temperature,
|
|
||||||
"stream": False,
|
|
||||||
}
|
|
||||||
data = json.dumps(payload).encode("utf-8")
|
|
||||||
headers = {"Content-Type": "application/json", "Content-Length": str(len(data))}
|
|
||||||
|
|
||||||
result = RequestResult(request_id=request_id, target_port=port)
|
|
||||||
start_time = time.perf_counter()
|
|
||||||
|
|
||||||
try:
|
|
||||||
req = urllib.request.Request(url, data=data, headers=headers, method="POST")
|
|
||||||
with urllib.request.urlopen(req, timeout=600) as resp:
|
|
||||||
body = json.loads(resp.read().decode("utf-8"))
|
|
||||||
|
|
||||||
choices = body.get("choices", [])
|
|
||||||
if not choices:
|
|
||||||
result.error = "no choices in response"
|
|
||||||
return result
|
|
||||||
|
|
||||||
content = choices[0].get("message", {}).get("content", "")
|
|
||||||
result.output_tokens = len(content.split()) if content else 0
|
|
||||||
result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000
|
|
||||||
result.first_token_time_ms = result.e2e_latency_ms
|
|
||||||
result.input_tokens = len(prompt.split())
|
|
||||||
result.success = True
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
result.error = str(e)[:200]
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Unified send_request wrapper with retry & failover
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def send_request_with_retry(
|
|
||||||
session: aiohttp.ClientSession | None,
|
|
||||||
host: str,
|
|
||||||
lb: LoadBalancer,
|
|
||||||
model: str,
|
|
||||||
request_id: int,
|
|
||||||
prompt: str,
|
|
||||||
max_tokens: int,
|
|
||||||
temperature: float,
|
|
||||||
max_retries: int = 2,
|
|
||||||
) -> RequestResult:
|
|
||||||
"""Send a request, retrying on failure with a different port each time."""
|
|
||||||
result = RequestResult(request_id=request_id)
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
port = await lb.next_port()
|
|
||||||
if HAS_AIOHTTP and session is not None:
|
|
||||||
result = await send_request_aiohttp(
|
|
||||||
session, host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
result = await loop.run_in_executor(
|
|
||||||
None, send_request_urllib,
|
|
||||||
host, port, model, request_id, prompt, max_tokens, temperature
|
|
||||||
)
|
|
||||||
result.retry_count = attempt
|
|
||||||
if result.success:
|
|
||||||
return result
|
|
||||||
# If failed, try another port on next iteration (unless last attempt).
|
|
||||||
if attempt < max_retries:
|
|
||||||
await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Benchmark orchestration
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def run_benchmark(
|
|
||||||
host: str,
|
|
||||||
ports: list[int],
|
|
||||||
lb: LoadBalancer,
|
|
||||||
health_checker: HealthChecker,
|
|
||||||
model: str,
|
|
||||||
concurrency: int,
|
|
||||||
input_len: int,
|
|
||||||
output_len: int,
|
|
||||||
num_prompts: int,
|
|
||||||
temperature: float,
|
|
||||||
output_file: str,
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Run the benchmark and write a JSONL file."""
|
|
||||||
prompts = [make_prompt(input_len) for _ in range(num_prompts)]
|
|
||||||
results: list[RequestResult] = []
|
|
||||||
semaphore = asyncio.Semaphore(concurrency)
|
|
||||||
|
|
||||||
start = time.perf_counter()
|
|
||||||
|
|
||||||
if HAS_AIOHTTP:
|
|
||||||
connector = aiohttp.TCPConnector(limit=concurrency * 2)
|
|
||||||
timeout = aiohttp.ClientTimeout(total=600, sock_read=300)
|
|
||||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
|
||||||
await health_checker.start(session)
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
session, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
else:
|
|
||||||
await health_checker.start(None)
|
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
|
|
||||||
async def bounded_send(idx: int, prompt: str) -> RequestResult:
|
|
||||||
async with semaphore:
|
|
||||||
return await send_request_with_retry(
|
|
||||||
None, host, lb, model, idx, prompt, output_len, temperature
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [bounded_send(i, p) for i, p in enumerate(prompts)]
|
|
||||||
results = await asyncio.gather(*tasks)
|
|
||||||
await health_checker.stop()
|
|
||||||
|
|
||||||
duration = time.perf_counter() - start
|
|
||||||
|
|
||||||
# Write JSONL
|
|
||||||
with open(output_file, "w", encoding="utf-8") as f:
|
|
||||||
for r in results:
|
|
||||||
f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
# Compute summary
|
|
||||||
successful = [r for r in results if r.success]
|
|
||||||
failed = len(results) - len(successful)
|
|
||||||
retried = [r for r in results if r.retry_count > 0]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
ttfts = [r.first_token_time_ms for r in successful]
|
|
||||||
tpots = [r.tpot_ms for r in successful if r.output_tokens > 1]
|
|
||||||
e2es = [r.e2e_latency_ms for r in successful]
|
|
||||||
itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies]
|
|
||||||
|
|
||||||
total_input_tokens = sum(r.input_tokens for r in successful)
|
|
||||||
total_output_tokens = sum(r.output_tokens for r in successful)
|
|
||||||
|
|
||||||
# Per-port breakdown
|
|
||||||
per_port_stats: dict[int, dict[str, Any]] = {}
|
|
||||||
for port in ports:
|
|
||||||
port_results = [r for r in successful if r.target_port == port]
|
|
||||||
port_failed = [r for r in results if r.target_port == port and not r.success]
|
|
||||||
if port_results or port_failed:
|
|
||||||
port_ttfts = [r.first_token_time_ms for r in port_results]
|
|
||||||
port_e2es = [r.e2e_latency_ms for r in port_results]
|
|
||||||
per_port_stats[port] = {
|
|
||||||
"count": len(port_results),
|
|
||||||
"failed": len(port_failed),
|
|
||||||
"mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0,
|
|
||||||
"p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0,
|
|
||||||
"mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Health-check stats
|
|
||||||
healthy_at_end = await health_checker.healthy_ports
|
|
||||||
|
|
||||||
summary = {
|
|
||||||
"completed": len(successful),
|
|
||||||
"failed": failed,
|
|
||||||
"retried": len(retried),
|
|
||||||
"duration": duration,
|
|
||||||
"request_throughput": len(successful) / duration if duration > 0 else 0.0,
|
|
||||||
"input_throughput": total_input_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"output_throughput": total_output_tokens / duration if duration > 0 else 0.0,
|
|
||||||
"total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0,
|
|
||||||
"total_input_tokens": total_input_tokens,
|
|
||||||
"total_output_tokens": total_output_tokens,
|
|
||||||
"mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0,
|
|
||||||
"median_ttft_ms": percentile(ttfts, 50),
|
|
||||||
"p90_ttft_ms": percentile(ttfts, 90),
|
|
||||||
"p95_ttft_ms": percentile(ttfts, 95),
|
|
||||||
"p99_ttft_ms": percentile(ttfts, 99),
|
|
||||||
"mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0,
|
|
||||||
"median_tpot_ms": percentile(tpots, 50),
|
|
||||||
"p90_tpot_ms": percentile(tpots, 90),
|
|
||||||
"p95_tpot_ms": percentile(tpots, 95),
|
|
||||||
"p99_tpot_ms": percentile(tpots, 99),
|
|
||||||
"mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0,
|
|
||||||
"median_e2e_latency_ms": percentile(e2es, 50),
|
|
||||||
"p90_e2e_latency_ms": percentile(e2es, 90),
|
|
||||||
"p95_e2e_latency_ms": percentile(e2es, 95),
|
|
||||||
"p99_e2e_latency_ms": percentile(e2es, 99),
|
|
||||||
"mean_itl_ms": sum(itls) / len(itls) if itls else 0.0,
|
|
||||||
"median_itl_ms": percentile(itls, 50),
|
|
||||||
"p90_itl_ms": percentile(itls, 90),
|
|
||||||
"p95_itl_ms": percentile(itls, 95),
|
|
||||||
"p99_itl_ms": percentile(itls, 99),
|
|
||||||
"per_port_stats": per_port_stats,
|
|
||||||
"healthy_ports_at_end": healthy_at_end,
|
|
||||||
"lb_strategy": lb.strategy,
|
|
||||||
"num_services": lb.port_count,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Append summary as the last line
|
|
||||||
with open(output_file, "a", encoding="utf-8") as f:
|
|
||||||
f.write(json.dumps(summary, ensure_ascii=False) + "\n")
|
|
||||||
|
|
||||||
return summary
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# CLI
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover")
|
|
||||||
parser.add_argument("--host", default="127.0.0.1")
|
|
||||||
parser.add_argument("--port", type=int, default=30005,
|
|
||||||
help="Single service port (backward compatible)")
|
|
||||||
parser.add_argument("--ports", default="",
|
|
||||||
help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008")
|
|
||||||
parser.add_argument("--model", default="deepseek-v4-flash")
|
|
||||||
parser.add_argument("--concurrency", type=int, default=1)
|
|
||||||
parser.add_argument("--input-len", type=int, default=512)
|
|
||||||
parser.add_argument("--output-len", type=int, default=256)
|
|
||||||
parser.add_argument("--num-prompts", type=int, default=10)
|
|
||||||
parser.add_argument("--temperature", type=float, default=0.0)
|
|
||||||
parser.add_argument("--lb-strategy", default="round_robin",
|
|
||||||
choices=["round_robin", "random", "least_conn"],
|
|
||||||
help="Load balancing strategy across services")
|
|
||||||
parser.add_argument("--health-check-interval", type=float, default=5.0,
|
|
||||||
help="Seconds between health checks (default: 5)")
|
|
||||||
parser.add_argument("--health-max-failures", type=int, default=2,
|
|
||||||
help="Consecutive failures before marking a port unhealthy (default: 2)")
|
|
||||||
parser.add_argument("--health-recovery-interval", type=float, default=10.0,
|
|
||||||
help="Seconds before retrying an unhealthy port (default: 10)")
|
|
||||||
parser.add_argument("--request-max-retries", type=int, default=2,
|
|
||||||
help="Max retries per request on failure (default: 2)")
|
|
||||||
parser.add_argument("--output-file", required=True)
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Determine ports: --ports takes precedence, else fall back to --port.
|
|
||||||
if args.ports:
|
|
||||||
ports = [int(p.strip()) for p in args.ports.split(",")]
|
|
||||||
else:
|
|
||||||
ports = [args.port]
|
|
||||||
|
|
||||||
health_checker = HealthChecker(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
check_interval=args.health_check_interval,
|
|
||||||
max_failures=args.health_max_failures,
|
|
||||||
recovery_interval=args.health_recovery_interval,
|
|
||||||
)
|
|
||||||
lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy)
|
|
||||||
|
|
||||||
summary = asyncio.run(run_benchmark(
|
|
||||||
host=args.host,
|
|
||||||
ports=ports,
|
|
||||||
lb=lb,
|
|
||||||
health_checker=health_checker,
|
|
||||||
model=args.model,
|
|
||||||
concurrency=args.concurrency,
|
|
||||||
input_len=args.input_len,
|
|
||||||
output_len=args.output_len,
|
|
||||||
num_prompts=args.num_prompts,
|
|
||||||
temperature=args.temperature,
|
|
||||||
output_file=args.output_file,
|
|
||||||
))
|
|
||||||
|
|
||||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,83 +0,0 @@
|
|||||||
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
|
|
||||||
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
|
|
||||||
#
|
|
||||||
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
|
|
||||||
# Service 1: port 30005, GPUs 0,1
|
|
||||||
# Service 2: port 30006, GPUs 2,3
|
|
||||||
# Service 3: port 30007, GPUs 4,5
|
|
||||||
# Service 4: port 30008, GPUs 6,7
|
|
||||||
#
|
|
||||||
# bench_client.py distributes requests across all 4 services via round-robin
|
|
||||||
# load balancing, giving accurate cluster-wide throughput even at low
|
|
||||||
# per-service concurrency.
|
|
||||||
|
|
||||||
EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench_no_fp8"
|
|
||||||
MODEL_NAME="DeepSeek-V4-Flash"
|
|
||||||
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
||||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
||||||
|
|
||||||
# Primary port (backward compatible); multi-service ports are in PORTS.
|
|
||||||
PORT="30005"
|
|
||||||
PORTS="30005"
|
|
||||||
|
|
||||||
BACKEND="vllm"
|
|
||||||
ENGINE="vllm"
|
|
||||||
|
|
||||||
# Native virtual environments on the host.
|
|
||||||
VENV_SERVER="/data/user1/yy/envs/vllm"
|
|
||||||
VENV_CLIENT="/data/user1/yy/envs/vllm"
|
|
||||||
|
|
||||||
# Load balancing strategy: round_robin | random | least_conn
|
|
||||||
LB_STRATEGY="round_robin"
|
|
||||||
|
|
||||||
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
|
||||||
# For single-service TP=8 we cover standard concurrency gradients.
|
|
||||||
SCENARIOS=(
|
|
||||||
"1 512 256 20"
|
|
||||||
"2 512 256 40"
|
|
||||||
"4 512 256 80"
|
|
||||||
"8 512 256 160"
|
|
||||||
"16 512 256 320"
|
|
||||||
"32 512 256 640"
|
|
||||||
"64 512 256 1280"
|
|
||||||
"128 512 256 2560"
|
|
||||||
"1 2048 512 20"
|
|
||||||
"8 2048 512 160"
|
|
||||||
"32 2048 512 640"
|
|
||||||
"128 2048 512 2560"
|
|
||||||
"1 4096 1024 20"
|
|
||||||
"8 4096 1024 160"
|
|
||||||
"32 4096 1024 640"
|
|
||||||
"128 4096 1024 2560"
|
|
||||||
"1 8192 1024 20"
|
|
||||||
"8 8192 1024 160"
|
|
||||||
"32 8192 1024 640"
|
|
||||||
"128 8192 1024 2560"
|
|
||||||
"1 16384 1024 20"
|
|
||||||
"8 16384 1024 160"
|
|
||||||
"32 16384 1024 640"
|
|
||||||
"128 16384 1024 2560"
|
|
||||||
"1 32768 1024 20"
|
|
||||||
"8 32768 1024 160"
|
|
||||||
"32 32768 1024 640"
|
|
||||||
"128 32768 1024 2560"
|
|
||||||
"1 65536 1024 20"
|
|
||||||
"8 65536 1024 160"
|
|
||||||
"32 65536 1024 640"
|
|
||||||
"128 65536 1024 2560"
|
|
||||||
"1 131072 1024 20"
|
|
||||||
"8 131072 1024 160"
|
|
||||||
"32 131072 1024 640"
|
|
||||||
"128 131072 1024 2560"
|
|
||||||
"1 262144 1024 20"
|
|
||||||
"8 262144 1024 160"
|
|
||||||
"32 262144 1024 640"
|
|
||||||
"128 262144 1024 2560"
|
|
||||||
"1 524288 1024 20"
|
|
||||||
"8 524288 1024 160"
|
|
||||||
"32 524288 1024 640"
|
|
||||||
"128 524288 1024 2560"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Server start script bundled with this experiment.
|
|
||||||
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"
|
|
||||||
@ -1,206 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark.
|
|
||||||
|
|
||||||
Reads JSONL files produced by bench_client.py (one request per line,
|
|
||||||
last line is the summary) and generates:
|
|
||||||
- results.json (appended scenarios, following the standard schema)
|
|
||||||
- report.md (human-readable summary)
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python3 parse_results.py <result_root>
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]:
|
|
||||||
"""Read the summary JSON (last line) and all per-request lines."""
|
|
||||||
requests: list[dict] = []
|
|
||||||
summary: dict | None = None
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
for line in f:
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
obj = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
# The summary line has "completed" and "duration" keys.
|
|
||||||
if "completed" in obj and "duration" in obj:
|
|
||||||
summary = obj
|
|
||||||
else:
|
|
||||||
requests.append(obj)
|
|
||||||
return summary, requests
|
|
||||||
|
|
||||||
|
|
||||||
def compute_metrics(summary: dict, requests: list[dict]) -> dict:
|
|
||||||
completed = summary.get("completed", 0)
|
|
||||||
total = len(requests)
|
|
||||||
failed = total - completed if total > 0 else 0
|
|
||||||
duration_s = summary.get("duration", 0.0)
|
|
||||||
|
|
||||||
# Extract per-request metrics for percentiles.
|
|
||||||
ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")]
|
|
||||||
tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1]
|
|
||||||
e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")]
|
|
||||||
itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")]
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
s = sorted(values)
|
|
||||||
k = (len(s) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(s) - 1)
|
|
||||||
return s[f] + (k - f) * (s[c] - s[f])
|
|
||||||
|
|
||||||
return {
|
|
||||||
"success": completed,
|
|
||||||
"failed": failed,
|
|
||||||
"duration_s": duration_s,
|
|
||||||
"request_throughput": summary.get("request_throughput", 0.0),
|
|
||||||
"input_token_throughput": summary.get("input_throughput", 0.0),
|
|
||||||
"output_token_throughput": summary.get("output_throughput", 0.0),
|
|
||||||
"total_token_throughput": summary.get("total_throughput", 0.0),
|
|
||||||
"total_input_tokens": summary.get("total_input_tokens", 0),
|
|
||||||
"total_output_tokens": summary.get("total_output_tokens", 0),
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": summary.get("mean_e2e_latency_ms", 0.0),
|
|
||||||
"p50": percentile(e2es, 50),
|
|
||||||
"p90": percentile(e2es, 90),
|
|
||||||
"p95": percentile(e2es, 95),
|
|
||||||
"p99": percentile(e2es, 99),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": summary.get("mean_ttft_ms", 0.0),
|
|
||||||
"p50": percentile(ttfts, 50),
|
|
||||||
"p90": percentile(ttfts, 90),
|
|
||||||
"p95": percentile(ttfts, 95),
|
|
||||||
"p99": percentile(ttfts, 99),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": summary.get("mean_tpot_ms", 0.0),
|
|
||||||
"p50": percentile(tpots, 50),
|
|
||||||
"p90": percentile(tpots, 90),
|
|
||||||
"p95": percentile(tpots, 95),
|
|
||||||
"p99": percentile(tpots, 99),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": summary.get("mean_itl_ms", 0.0),
|
|
||||||
"p50": percentile(itls, 50),
|
|
||||||
"p90": percentile(itls, 90),
|
|
||||||
"p95": percentile(itls, 95),
|
|
||||||
"p99": percentile(itls, 99),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
|
|
||||||
return f"c{concurrency}_i{input_len}_o{output_len}"
|
|
||||||
|
|
||||||
|
|
||||||
def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict:
|
|
||||||
ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms
|
|
||||||
tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms
|
|
||||||
if ttft_ok and tpot_ok:
|
|
||||||
mark = "✅"
|
|
||||||
elif ttft_ok or tpot_ok:
|
|
||||||
mark = "⚠️"
|
|
||||||
else:
|
|
||||||
mark = "❌"
|
|
||||||
return {
|
|
||||||
"ttft_p95_ok": ttft_ok,
|
|
||||||
"tpot_mean_ok": tpot_ok,
|
|
||||||
"overall": mark,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def append_scenario(results_json: Path, scenario: dict) -> None:
|
|
||||||
with open(results_json, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
data["scenarios"].append(scenario)
|
|
||||||
with open(results_json, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
|
|
||||||
def generate_report(result_root: Path, scenarios: list[dict]) -> None:
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n")
|
|
||||||
f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n")
|
|
||||||
f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n")
|
|
||||||
f.write(f"- **Result root**: `{result_root}`\n\n")
|
|
||||||
|
|
||||||
f.write("## Results\n\n")
|
|
||||||
f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
|
||||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
|
||||||
|
|
||||||
for s in scenarios:
|
|
||||||
cfg = s["config"]
|
|
||||||
m = s["metrics"]
|
|
||||||
slo = s.get("slo_status", {}).get("overall", "")
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
|
||||||
f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | "
|
|
||||||
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
|
||||||
f"{m['total_token_throughput']:.2f} | "
|
|
||||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
|
||||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
|
||||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
|
||||||
)
|
|
||||||
f.write("\n")
|
|
||||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
results_json = result_root / "results.json"
|
|
||||||
|
|
||||||
if not raw_dir.exists():
|
|
||||||
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")):
|
|
||||||
parts = jsonl_path.stem.split("_")
|
|
||||||
if len(parts) < 5:
|
|
||||||
continue
|
|
||||||
concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4])
|
|
||||||
|
|
||||||
summary, requests = parse_jsonl(jsonl_path)
|
|
||||||
if summary is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
metrics = compute_metrics(summary, requests)
|
|
||||||
scenario = {
|
|
||||||
"name": scenario_name(concurrency, input_len, output_len),
|
|
||||||
"config": {
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"dataset": "random",
|
|
||||||
"num_prompts": metrics["success"] + metrics["failed"],
|
|
||||||
},
|
|
||||||
"metrics": metrics,
|
|
||||||
"slo_status": slo_status(metrics),
|
|
||||||
"raw_file": str(jsonl_path),
|
|
||||||
}
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
if not scenarios:
|
|
||||||
print("No benchmark outputs found to parse")
|
|
||||||
return
|
|
||||||
|
|
||||||
if results_json.exists():
|
|
||||||
for s in scenarios:
|
|
||||||
append_scenario(results_json, s)
|
|
||||||
|
|
||||||
generate_report(result_root, scenarios)
|
|
||||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,272 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200.
|
|
||||||
#
|
|
||||||
# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7).
|
|
||||||
# bench_client.py sends requests to the single service.
|
|
||||||
#
|
|
||||||
# Usage:
|
|
||||||
# bash run_bench.sh
|
|
||||||
# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
|
||||||
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
||||||
RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
|
|
||||||
RAW_DIR="${RESULT_ROOT}/raw_outputs"
|
|
||||||
LOG_DIR="${RESULT_ROOT}/logs"
|
|
||||||
|
|
||||||
ensure_result_root "$RESULT_ROOT"
|
|
||||||
log_init "${LOG_DIR}/orchestrator.log"
|
|
||||||
|
|
||||||
log "experiment=${EXPERIMENT_NAME}"
|
|
||||||
log "run_id=${RUN_ID}"
|
|
||||||
log "result_root=${RESULT_ROOT}"
|
|
||||||
log "platform=${PLATFORM}"
|
|
||||||
log "chip=${CHIP}"
|
|
||||||
log "accelerator=${ACCELERATOR}"
|
|
||||||
log "engine=${ENGINE}"
|
|
||||||
log "hardware=${HARDWARE}"
|
|
||||||
log "model=${MODEL_PATH}"
|
|
||||||
log "services=${PORTS}"
|
|
||||||
log "lb_strategy=${LB_STRATEGY}"
|
|
||||||
log "server_start_script=${SERVER_START_SCRIPT}"
|
|
||||||
|
|
||||||
# Write initial metadata for this run.
|
|
||||||
METADATA_JSON="${RESULT_ROOT}/results.json"
|
|
||||||
write_metadata_json \
|
|
||||||
"$METADATA_JSON" \
|
|
||||||
"$EXPERIMENT_NAME" \
|
|
||||||
"$RUN_ID" \
|
|
||||||
"$MODEL_PATH" \
|
|
||||||
"$BACKEND" \
|
|
||||||
"$ENGINE" \
|
|
||||||
"$HARDWARE" \
|
|
||||||
"$ACCELERATOR" \
|
|
||||||
"$CHIP" \
|
|
||||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
|
||||||
"$VENV_SERVER" \
|
|
||||||
"H200 1x vLLM TP=8 single-service (no FP8 KV cache) benchmark using bench_client.py"
|
|
||||||
|
|
||||||
# Update metadata with config.
|
|
||||||
"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY'
|
|
||||||
import json, sys
|
|
||||||
path = sys.argv[1]
|
|
||||||
with open(path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
data["config"] = {
|
|
||||||
"tp": 8,
|
|
||||||
"num_services": 1,
|
|
||||||
"ports": "30005",
|
|
||||||
"gpu_groups": "0,1,2,3,4,5,6,7",
|
|
||||||
"kv_cache_dtype": "fp8",
|
|
||||||
"block_size": 256,
|
|
||||||
"max_num_seqs": 256,
|
|
||||||
"client": "bench_client.py",
|
|
||||||
"lb_strategy": "round_robin",
|
|
||||||
"scenarios": [
|
|
||||||
"1 512 256 20", "2 512 256 40", "4 512 256 80",
|
|
||||||
"8 512 256 160", "16 512 256 320", "32 512 256 640",
|
|
||||||
"64 512 256 1280", "128 512 256 2560",
|
|
||||||
"1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560",
|
|
||||||
"1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
with open(path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
PY
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Server helpers (single-service)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
is_server_healthy() {
|
|
||||||
local port="$1"
|
|
||||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
are_all_services_healthy() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if ! is_server_healthy "$p"; then
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_all_servers() {
|
|
||||||
log "stopping all vllm services"
|
|
||||||
# Kill by PID files first.
|
|
||||||
for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do
|
|
||||||
[[ -f "$pid_file" ]] || continue
|
|
||||||
local pid
|
|
||||||
pid="$(cat "$pid_file")"
|
|
||||||
if kill -0 "$pid" 2>/dev/null; then
|
|
||||||
log "killing pid=${pid}"
|
|
||||||
kill "$pid" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
kill -9 "$pid" 2>/dev/null || true
|
|
||||||
fi
|
|
||||||
rm -f "$pid_file"
|
|
||||||
done
|
|
||||||
# Fallback: kill any remaining vllm processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
}
|
|
||||||
|
|
||||||
start_all_servers() {
|
|
||||||
log "starting all vllm services with ${SERVER_START_SCRIPT}"
|
|
||||||
if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then
|
|
||||||
log "error: start script not found or not executable: ${SERVER_START_SCRIPT}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1
|
|
||||||
|
|
||||||
if ! are_all_services_healthy "$PORTS"; then
|
|
||||||
log "error: not all vllm services became healthy"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
log "all vllm services are healthy"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Pre-benchmark health check with detailed reporting
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_services_detailed() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy_count=0
|
|
||||||
local unhealthy_ports=""
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
((healthy_count++))
|
|
||||||
else
|
|
||||||
unhealthy_ports="${unhealthy_ports}${p} "
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo "$healthy_count"
|
|
||||||
if [[ -n "$unhealthy_ports" ]]; then
|
|
||||||
echo "unhealthy: ${unhealthy_ports}" >&2
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_all_services() {
|
|
||||||
local ports="$1"
|
|
||||||
local max_wait="${2:-240}"
|
|
||||||
local interval="${3:-5}"
|
|
||||||
local elapsed=0
|
|
||||||
while (( elapsed < max_wait )); do
|
|
||||||
local healthy_count
|
|
||||||
healthy_count="$(check_services_detailed "$ports" | head -1)"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
if (( healthy_count == ${#PORT_LIST[@]} )); then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s"
|
|
||||||
sleep "$interval"
|
|
||||||
((elapsed += interval))
|
|
||||||
done
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Mid-benchmark health check (called between scenarios)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_and_report_services() {
|
|
||||||
local ports="$1"
|
|
||||||
IFS=',' read -ra PORT_LIST <<< "$ports"
|
|
||||||
local healthy=()
|
|
||||||
local unhealthy=()
|
|
||||||
for p in "${PORT_LIST[@]}"; do
|
|
||||||
if is_server_healthy "$p"; then
|
|
||||||
healthy+=("$p")
|
|
||||||
else
|
|
||||||
unhealthy+=("$p")
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]"
|
|
||||||
if [[ ${#unhealthy[@]} -gt 0 ]]; then
|
|
||||||
log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}"
|
|
||||||
fi
|
|
||||||
return ${#unhealthy[@]}
|
|
||||||
}
|
|
||||||
|
|
||||||
on_exit() {
|
|
||||||
local code=$?
|
|
||||||
log "orchestrator exiting with code=${code}"
|
|
||||||
if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
stop_all_servers
|
|
||||||
fi
|
|
||||||
exit "$code"
|
|
||||||
}
|
|
||||||
trap on_exit EXIT
|
|
||||||
|
|
||||||
if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then
|
|
||||||
log "SKIP_MANAGE_SERVER is set, assuming services are already running"
|
|
||||||
if ! wait_for_all_services "$PORTS"; then
|
|
||||||
log "error: not all healthy services found on ports ${PORTS}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
stop_all_servers
|
|
||||||
start_all_servers
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "===== BENCHMARK START ====="
|
|
||||||
|
|
||||||
for scenario in "${SCENARIOS[@]}"; do
|
|
||||||
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
|
||||||
# Fallback for legacy 3-field scenarios.
|
|
||||||
if [[ -z "${num_prompts:-}" ]]; then
|
|
||||||
num_prompts="$NUM_PROMPTS"
|
|
||||||
fi
|
|
||||||
|
|
||||||
output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
|
||||||
detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log"
|
|
||||||
|
|
||||||
log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}"
|
|
||||||
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \
|
|
||||||
--host 127.0.0.1 \
|
|
||||||
--port "$PORT" \
|
|
||||||
--model "$SERVED_MODEL_NAME" \
|
|
||||||
--concurrency "$concurrency" \
|
|
||||||
--input-len "$input_len" \
|
|
||||||
--output-len "$output_len" \
|
|
||||||
--num-prompts "$num_prompts" \
|
|
||||||
--output-file "$output_file" \
|
|
||||||
> "$detail_log" 2>&1 || {
|
|
||||||
log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
|
||||||
# After failure, check service health before continuing.
|
|
||||||
check_and_report_services "$PORTS"
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
log "finished scenario: output=${output_file}"
|
|
||||||
|
|
||||||
# Between scenarios, report service health so we can spot degradation early.
|
|
||||||
check_and_report_services "$PORTS" || true
|
|
||||||
done
|
|
||||||
|
|
||||||
log "===== BENCHMARK DONE ====="
|
|
||||||
|
|
||||||
log "parsing results"
|
|
||||||
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || {
|
|
||||||
log "WARNING: parser failed; see ${LOG_DIR}/parse.log"
|
|
||||||
}
|
|
||||||
|
|
||||||
log "all results saved to ${RESULT_ROOT}"
|
|
||||||
@ -1,106 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Start 1 vLLM TP=8 service on 8x H200.
|
|
||||||
# Uses all 8 GPUs: (0,1,2,3,4,5,6,7).
|
|
||||||
# This maximizes single-service throughput for highest-concurrency workloads.
|
|
||||||
set -e
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
||||||
# shellcheck source=/dev/null
|
|
||||||
source "${SCRIPT_DIR}/config.env"
|
|
||||||
|
|
||||||
VENV="${VENV_SERVER}"
|
|
||||||
export PATH="$VENV/bin:$PATH"
|
|
||||||
VLLM="$VENV/bin/vllm"
|
|
||||||
|
|
||||||
TP=8
|
|
||||||
LOG_DIR="${SCRIPT_DIR}/logs"
|
|
||||||
mkdir -p "$LOG_DIR"
|
|
||||||
|
|
||||||
# 1 service on all 8 GPUs.
|
|
||||||
# Port and GPU mapping.
|
|
||||||
SERVICES=(
|
|
||||||
"30005:0,1,2,3,4,5,6,7"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Kill any existing vLLM processes for this model.
|
|
||||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
# Remove old PID files.
|
|
||||||
rm -f "${SCRIPT_DIR}"/*.pid
|
|
||||||
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
GPUS="${svc##*:}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
|
|
||||||
echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ==="
|
|
||||||
echo "Log: ${LOG}"
|
|
||||||
|
|
||||||
CUDA_VISIBLE_DEVICES="${GPUS}" \
|
|
||||||
nohup "$VLLM" serve "$MODEL_PATH" \
|
|
||||||
--trust-remote-code \
|
|
||||||
--tensor-parallel-size "$TP" \
|
|
||||||
--block-size 256 \
|
|
||||||
--served-model-name "$SERVED_MODEL_NAME" \
|
|
||||||
--port "$PORT" \
|
|
||||||
> "$LOG" 2>&1 &
|
|
||||||
PID=$!
|
|
||||||
echo $PID > "$PID_FILE"
|
|
||||||
echo "PID: $PID"
|
|
||||||
done
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "Waiting for service to become healthy..."
|
|
||||||
|
|
||||||
MAX_WAIT=240
|
|
||||||
all_healthy=0
|
|
||||||
for i in $(seq 1 $MAX_WAIT); do
|
|
||||||
all_healthy=1
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then
|
|
||||||
all_healthy=0
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$all_healthy" -eq 1 ]]; then
|
|
||||||
echo "Service is healthy!"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
echo " - http://127.0.0.1:${PORT}"
|
|
||||||
done
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if any process died early.
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid"
|
|
||||||
if [[ -f "$PID_FILE" ]]; then
|
|
||||||
PID="$(cat "$PID_FILE")"
|
|
||||||
if ! kill -0 "$PID" 2>/dev/null; then
|
|
||||||
echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
tail -200 $LOG 2>/dev/null || true
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
if (( i % 10 == 0 )); then
|
|
||||||
echo "Waiting... (${i}/${MAX_WAIT})"
|
|
||||||
fi
|
|
||||||
sleep 5
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries"
|
|
||||||
for svc in "${SERVICES[@]}"; do
|
|
||||||
PORT="${svc%%:*}"
|
|
||||||
LOG="${LOG_DIR}/server_port${PORT}_*.log"
|
|
||||||
echo "--- Last 50 lines of port ${PORT} ---"
|
|
||||||
tail -50 $LOG 2>/dev/null || true
|
|
||||||
done
|
|
||||||
exit 1
|
|
||||||
Loading…
x
Reference in New Issue
Block a user