From 372ff08eea3c36d8a63b5bc3c32bac1e67b42b8f Mon Sep 17 00:00:00 2001 From: Quantong Qiu Date: Mon, 13 Jul 2026 02:46:15 +0000 Subject: [PATCH] Add master orchestrator script for sequential benchmark execution - Implemented a bash script to run six benchmark experiments sequentially. - Added features for fault tolerance, out-of-memory recovery, and checkpoint management. - Included detailed logging for monitoring experiment progress and results. - Integrated GPU memory cleanup checks before each experiment. - Provided functionality to resume experiments from checkpoints. --- .../bench_client.py | 53 +- .../config.env | 184 +- .../results/20260710-091522/results.json | 47 - .../results/20260710-091700/results.json | 47 - .../results.json | 6 +- .../README.md | 273 - .../bench_client.py | 617 -- .../config.env | 84 - .../parse_results.py | 206 - .../results/20260710-091141/results.json | 47 - .../results/20260710-091522/results.json | 47 - .../results/20260710-091700/results.json | 47 - .../run_bench.sh | 277 - .../start_server.sh | 109 - .../bench_client.py | 53 +- .../config.env | 184 +- .../results/20260711-133908/report.md | 129 + .../results/20260711-133908/results.json | 6429 +++++++++++++++++ .../README.md | 126 - .../bench_client.py | 617 -- .../config.env | 84 - .../parse_results.py | 206 - .../run_bench.sh | 278 - .../start_server.sh | 107 - .../bench_client.py | 53 +- .../config.env | 185 +- .../results/20260711-193756/report.md | 129 + .../results/20260711-193756/results.json | 6428 ++++++++++++++++ .../run_bench.sh | 1 - .../README.md | 103 - .../bench_client.py | 617 -- .../config.env | 83 - .../parse_results.py | 206 - .../run_bench.sh | 272 - .../start_server.sh | 106 - 35 files changed, 13671 insertions(+), 4769 deletions(-) delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json rename experiments/dsv4_h200_vllm_tp2_custom_bench/results/{20260710-091141 => 20260710-104930}/results.json (91%) delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md create mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md create mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json delete mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py delete mode 100644 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh delete mode 100755 experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env index 2b2d56c..edee0e7 100644 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/config.env @@ -34,50 +34,146 @@ LB_STRATEGY="round_robin" # For multi-service we include low-concurrency scenarios to show how # requests spread across 4 services. SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json deleted file mode 100644 index 3196c06..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091522/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091522", - "timestamp": "2026-07-10T09:15:22+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json deleted file mode 100644 index 16c1892..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091700/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091700", - "timestamp": "2026-07-10T09:17:00+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json similarity index 91% rename from experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json rename to experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json index 64dd782..5c17b81 100644 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-091141/results.json +++ b/experiments/dsv4_h200_vllm_tp2_custom_bench/results/20260710-104930/results.json @@ -1,8 +1,8 @@ { "metadata": { "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091141", - "timestamp": "2026-07-10T09:11:41+00:00", + "run_id": "20260710-104930", + "timestamp": "2026-07-10T10:49:30+00:00", "model": "/data/models/DeepSeek-V4-Flash", "backend": "vllm", "engine": "vllm", @@ -11,7 +11,7 @@ "chip": "nvidia_h200", "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", + "git_commit": "def1355", "git_dirty": "dirty", "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" }, diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md deleted file mode 100644 index dae2993..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,273 +0,0 @@ -# dsv4_h200_vllm_tp2_custom_bench_no_fp8 - -> **Note:** This variant does **NOT** use `--kv-cache-dtype fp8`. - -## 目的 - -当 vLLM 使用 **TP=2**（Tensor Parallel = 2）部署时，单张 H200 上可以同时运行 **4 个独立服务**（8 卡 / 2 = 4 服务），每个服务独占 2 张 GPU： - -| 服务 | 端口 | GPU 对 | -|------|------|--------| -| 1 | 30005 | 0, 1 | -| 2 | 30006 | 2, 3 | -| 3 | 30007 | 4, 5 | -| 4 | 30008 | 6, 7 | - -此时 `sglang.bench_serving` 存在以下问题： - -1. **低并发下无法测出真实吞吐**：并发数不大时，请求可能只落在一个服务上，其他 3 个服务空闲，bench 报告的吞吐只是单服务的 1/4。 -2. **TTFT 测量不够精确**：`sglang.bench_serving` 的计时粒度较粗，对于低延迟场景（TP=2 下 TTFT 可能 < 100ms）误差较大。 -3. **不支持多服务负载均衡**：只能测一个端口，无法反映整个 8 卡集群性能。 - -因此本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API，通过 `aiohttp` 并发发送请求，并记录每个请求的精确时间戳。同时通过 **负载均衡** 将请求均匀分发到 4 个服务上。 - -## 与 sglang.bench_serving 的区别 - -| 特性 | sglang.bench_serving | bench_client.py (本实验) | -|------|----------------------|--------------------------| -| 协议 | 内部协议 | OpenAI API (`/v1/chat/completions`) | -| 并发控制 | 进程级 | `asyncio.Semaphore` | -| TTFT 测量 | 粗略 | 精确到 `time.perf_counter()` | -| TPOT 测量 | 基于总时间估算 | 基于首 token 后每个 token 的间隔 | -| 多服务支持 | ❌ 单端口 | ✅ 轮询/随机分发到多个端口 | -| 低并发精度 | 一般 | 高（适合 TP=2 场景） | -| 集群吞吐 | 只能测单服务 | 可测 4 服务总吞吐 | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡 | -| `config.env` | 实验配置（场景、模型路径、端口列表、LB 策略等） | -| `run_bench.sh` | 编排脚本：启动 4 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 同时启动 4 个 vLLM TP=2 服务（每服务 2 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动 4 个服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh - -# 跳过服务管理（已有 4 个服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh - -# 单独运行压测客户端（多服务，带健康检查） -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --lb-strategy round_robin \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file /tmp/test.jsonl - -# 单独运行压测客户端（单服务，向后兼容） -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ - --output-file /tmp/test_single.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp2_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp2_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 4 | 512 | 256 | 80 | 低并发，4 服务各 1 req | -| 8 | 512 | 256 | 160 | 中低并发 | -| 16 | 512 | 256 | 320 | 中等并发 | -| 32 | 512 | 256 | 640 | 中高并发 | -| 64 | 512 | 256 | 1280 | 高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 健康检查与故障自动跳过 - -`bench_client.py` 内置了 **后台健康检查 + 请求级重试** 机制，确保某个服务挂了不会导致整个 benchmark 失败。 - -### 健康检查机制 - -| 参数 | 默认值 | 说明 | -|------|--------|------| -| `--health-check-interval` | 5.0 | 每 N 秒检查一次所有服务 | -| `--health-max-failures` | 2 | 连续失败 N 次才标记为不健康 | -| `--health-recovery-interval` | 10.0 | 不健康服务 N 秒后自动重试恢复 | -| `--request-max-retries` | 2 | 单个请求失败最多重试 N 次 | - -### 故障处理流程 - -``` -请求 → 负载均衡选择端口 → 发送请求 - ↓ -成功 → 记录结果 - ↓ -失败 → 换另一个端口重试（最多 --request-max-retries 次） - ↓ -全部失败 → 记录失败，继续下一个请求 -``` - -后台同时运行健康检查： -- 每 5 秒对所有端口执行 `/health` 探测 -- 连续 2 次失败的服务被标记为 **unhealthy**，不再接收新请求 -- 10 秒后自动尝试恢复，如果恢复成功则重新加入负载均衡池 - -### 输出中的故障信息 - -汇总 JSON 包含以下字段： - -```json -{ - "completed": 638, - "failed": 2, - "retried": 5, - "healthy_ports_at_end": [30005, 30006, 30007], - "per_port_stats": { - "30005": {"count": 160, "failed": 0, ...}, - "30006": {"count": 159, "failed": 1, ...}, - "30007": {"count": 160, "failed": 0, ...}, - "30008": {"count": 159, "failed": 1, ...} - } -} -``` - -- `failed`: 最终失败的请求数（重试后仍失败） -- `retried`: 至少重试过一次的请求数 -- `healthy_ports_at_end`: 压测结束时仍健康的服务端口 -- `per_port_stats[].failed`: 每个端口的失败数 - -### 场景间健康检查 - -`run_bench.sh` 在每个场景结束后会打印一次服务健康状态： - -``` -[2026-07-10T08:45:00+00:00] service health check: healthy=[30005 30006 30007 30008] unhealthy=[] -[2026-07-10T08:45:30+00:00] service health check: healthy=[30005 30006 30007] unhealthy=[30008] -[2026-07-10T08:45:30+00:00] WARNING: 1 service(s) unhealthy: 30008 -``` - -这样即使某个服务在压测中途 OOM 崩溃，其他服务仍能继续工作，benchmark 不会中断。 - -### 自定义故障参数 - -```bash -python3 bench_client.py \ - --ports 30005,30006,30007,30008 \ - --health-check-interval 3 \ - --health-max-failures 1 \ - --health-recovery-interval 5 \ - --request-max-retries 3 \ - ... -``` - -- 更敏感：间隔 3 秒、1 次失败即标记不健康、5 秒恢复 -- 更容错：单个请求最多重试 3 次 - -## 负载均衡策略 - -`bench_client.py` 支持三种负载均衡策略： - -| 策略 | 说明 | 适用场景 | -|------|------|----------| -| `round_robin` | 请求按顺序轮询分配到各端口 | **默认**，最公平，推荐 | -| `random` | 随机选择一个端口 | 简单，分布可能不均 | -| `least_conn` | 选择当前连接数最少的端口 | 需要追踪 in-flight，当前为 random 占位 | - -通过 `--lb-strategy` 参数选择： - -```bash -python3 bench_client.py --lb-strategy round_robin --ports 30005,30006,30007,30008 ... -``` - -## 输出格式 - -### JSONL 原始数据 (`raw_outputs/vllm_*.jsonl`) - -每行一个 JSON 对象，包含请求分发到的目标端口： - -```json -{ - "request_id": 0, - "input_tokens": 512, - "output_tokens": 256, - "first_token_time_ms": 45.23, - "e2e_latency_ms": 5234.56, - "inter_token_latencies": [18.5, 19.2, ...], - "tpot_ms": 19.8, - "mean_itl_ms": 19.5, - "success": true, - "error": "", - "target_port": 30005 -} -``` - -最后一行是汇总数据，包含 **各端口统计**： - -```json -{ - "completed": 640, - "failed": 0, - "duration": 12.34, - "request_throughput": 51.88, - "mean_ttft_ms": 48.5, - "p95_ttft_ms": 62.3, - "per_port_stats": { - "30005": {"count": 160, "mean_ttft_ms": 47.2, "p95_ttft_ms": 58.1, "mean_e2e_ms": 5200.0}, - "30006": {"count": 160, "mean_ttft_ms": 49.1, "p95_ttft_ms": 64.3, "mean_e2e_ms": 5250.0}, - "30007": {"count": 160, "mean_ttft_ms": 48.8, "p95_ttft_ms": 61.5, "mean_e2e_ms": 5230.0}, - "30008": {"count": 160, "mean_ttft_ms": 48.9, "p95_ttft_ms": 65.2, "mean_e2e_ms": 5270.0} - }, - "lb_strategy": "round_robin", - "num_services": 4 -} -``` - -### 结构化结果 (`results.json`) - -遵循仓库统一的 [JSON Schema](../../BENCHMARK_WORKFLOW.md#final-json-schema)。 - -### 人类可读报告 (`report.md`) - -Markdown 表格，包含所有场景的 TTFT/TPOT/E2E 均值与分位数，以及 SLO 达标状态。 - -## 单独启动/停止 4 个服务 - -```bash -# 启动 4 个服务 -bash experiments/dsv4_h200_vllm_tp2_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -for port in 30005 30006 30007 30008; do - pid_file="experiments/dsv4_h200_vllm_tp2_custom_bench/server_port${port}.pid" - [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true - rm -f "$pid_file" -done -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- `bench_client.py` 优先使用 `aiohttp`，如果未安装则自动回退到 `urllib`（但会丢失 streaming 和精确 TTFT/ITL 测量）。 -- 默认 `temperature=0.0` 以保证输出长度稳定。 -- 输入 token 数通过空格分隔的单词数估算，实际 tokenizer 可能略有偏差。 -- 4 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 -- 每个服务占用约 2 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 -- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 -- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env deleted file mode 100644 index ddeb143..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,84 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp2_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005,30006,30007,30008" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For multi-service we include low-concurrency scenarios to show how -# requests spread across 4 services. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json deleted file mode 100644 index 64dd782..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091141/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091141", - "timestamp": "2026-07-10T09:11:41+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json deleted file mode 100644 index 3196c06..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091522/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091522", - "timestamp": "2026-07-10T09:15:22+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json deleted file mode 100644 index 16c1892..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/results/20260710-091700/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "metadata": { - "experiment": "dsv4_h200_vllm_tp2_custom_bench", - "run_id": "20260710-091700", - "timestamp": "2026-07-10T09:17:00+00:00", - "model": "/data/models/DeepSeek-V4-Flash", - "backend": "vllm", - "engine": "vllm", - "hardware": "8x NVIDIA H200 143GB", - "accelerator": "NVIDIA H200", - "chip": "nvidia_h200", - "script": "experiments/dsv4_h200_vllm_tp2_custom_bench/run_bench.sh", - "env": "/data/user1/yy/envs/vllm", - "git_commit": "39fb076", - "git_dirty": "dirty", - "description": "H200 4x vLLM TP=2 multi-service benchmark using bench_client.py" - }, - "config": { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", - "4 512 256 80", - "8 512 256 160", - "16 512 256 320", - "32 512 256 640", - "64 512 256 1280", - "128 512 256 2560", - "1 2048 512 20", - "8 2048 512 160", - "32 2048 512 640", - "128 2048 512 2560", - "1 4096 1024 20", - "8 4096 1024 160", - "32 4096 1024 640", - "128 4096 1024 2560" - ] - }, - "scenarios": [] -} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 6be99c8..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for multi-service vLLM TP=2 on 8x H200. -# -# This runs 4 independent vLLM services (TP=2 each) on GPU pairs (0,1),(2,3),(4,5),(6,7). -# bench_client.py distributes requests across all 4 services via round-robin LB, -# giving accurate cluster-wide throughput even at low per-service concurrency. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 4x vLLM TP=2 multi-service (no FP8 KV cache) benchmark using bench_client.py" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 2, - "num_services": 4, - "ports": "30005,30006,30007,30008", - "gpu_pairs": "0,1|2,3|4,5|6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "4 512 256 80", "8 512 256 160", - "16 512 256 320", "32 512 256 640", "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (multi-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --ports "$PORTS" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --lb-strategy "$LB_STRATEGY" \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index 8555596..0000000 --- a/experiments/dsv4_h200_vllm_tp2_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -# Start 4 independent vLLM TP=2 services on 8x H200. -# Each service runs on a pair of GPUs: (0,1), (2,3), (4,5), (6,7). -# This maximizes GPU utilization for low-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=2 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 4 services, each on 2 GPUs. -# Port and GPU pair mapping. -SERVICES=( - "30005:0,1" - "30006:2,3" - "30007:4,5" - "30008:6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for all 4 services to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "All 4 services are healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env index 365ef84..881531e 100644 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/config.env @@ -34,50 +34,146 @@ LB_STRATEGY="round_robin" # For multi-service we include low-concurrency scenarios to show how # requests spread across 2 services. SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md new file mode 100644 index 0000000..4355fae --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/report.md @@ -0,0 +1,129 @@ +# H200 vLLM TP=2 Custom Benchmark Report + +- **Client**: `bench_client.py` (async OpenAI API, per-request timing) +- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding) +- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i1024_o1024 | 128 | 1024 | 1024 | 172.39 | 2560 | 0 | 14.85 | 15206.17 | 3100.60 | 18306.76 | 267.16 | 478.80 | 2297.70 | 38.53 | 66.16 | 70.07 | 8265.00 | 13967.23 | 17536.46 | ✅ | +| c128_i1024_o128 | 128 | 1024 | 128 | 98.84 | 2560 | 0 | 25.90 | 26522.07 | 3314.91 | 29836.98 | 900.39 | 1590.87 | 2307.93 | 30.16 | 35.98 | 37.09 | 4730.89 | 5530.64 | 6122.15 | ✅ | +| c128_i1024_o2048 | 128 | 1024 | 2048 | 161.76 | 2560 | 0 | 15.83 | 16205.28 | 3297.74 | 19503.02 | 256.61 | 340.87 | 2315.99 | 36.07 | 39.27 | 40.30 | 7732.01 | 10616.59 | 11870.42 | ✅ | +| c128_i1024_o256 | 128 | 1024 | 256 | 161.16 | 2560 | 0 | 15.89 | 16266.52 | 3250.72 | 19517.24 | 259.69 | 356.25 | 2333.30 | 36.68 | 40.81 | 51.99 | 7727.17 | 10133.95 | 11225.21 | ✅ | +| c128_i1024_o4096 | 128 | 1024 | 4096 | 163.16 | 2560 | 0 | 15.69 | 16066.27 | 3281.03 | 19347.31 | 257.28 | 422.58 | 2303.92 | 36.09 | 39.21 | 40.35 | 7768.07 | 10730.88 | 12234.15 | ✅ | +| c128_i1024_o512 | 128 | 1024 | 512 | 161.49 | 2560 | 0 | 15.85 | 16232.48 | 3275.92 | 19508.40 | 255.16 | 334.67 | 2297.21 | 36.27 | 39.39 | 40.57 | 7716.22 | 10581.99 | 11832.60 | ✅ | +| c128_i16384_o1024 | 128 | 16384 | 1024 | 919.25 | 2560 | 0 | 2.78 | 45627.36 | 697.33 | 46324.69 | 2625.93 | 3630.11 | 33634.45 | 171.30 | 196.69 | 207.71 | 45001.23 | 90438.26 | 123707.69 | ❌ | +| c128_i16384_o128 | 128 | 16384 | 128 | 859.37 | 2560 | 0 | 2.98 | 48806.72 | 380.95 | 49187.68 | 2833.19 | 3131.25 | 33643.84 | 311.88 | 328.04 | 328.10 | 42403.99 | 44239.06 | 73426.74 | ❌ | +| c128_i16384_o2048 | 128 | 16384 | 2048 | 918.61 | 2560 | 0 | 2.79 | 45659.28 | 701.99 | 46361.27 | 2601.36 | 3607.43 | 33624.66 | 170.03 | 195.00 | 208.79 | 45030.71 | 92401.43 | 124284.89 | ❌ | +| c128_i16384_o256 | 128 | 16384 | 256 | 897.62 | 2560 | 0 | 2.85 | 46726.87 | 602.42 | 47329.29 | 2804.71 | 4596.59 | 33617.37 | 197.06 | 224.66 | 236.73 | 44219.84 | 57755.16 | 76587.71 | ❌ | +| c128_i16384_o512 | 128 | 16384 | 512 | 915.13 | 2560 | 0 | 2.80 | 45833.00 | 684.56 | 46517.56 | 2646.03 | 3737.57 | 33718.32 | 174.28 | 203.88 | 224.11 | 44914.54 | 88267.88 | 100313.27 | ❌ | +| c128_i2048_o1024 | 128 | 2048 | 1024 | 204.30 | 2560 | 0 | 12.53 | 25663.20 | 2671.39 | 28334.59 | 372.84 | 493.00 | 4208.03 | 44.24 | 48.11 | 50.53 | 9762.62 | 15935.84 | 21541.33 | ✅ | +| c128_i2048_o128 | 128 | 2048 | 128 | 153.29 | 2560 | 0 | 16.70 | 34202.45 | 2125.54 | 36327.99 | 791.19 | 1283.06 | 4199.42 | 52.51 | 58.15 | 59.97 | 7426.54 | 8226.85 | 10576.91 | ⚠️ | +| c128_i2048_o2048 | 128 | 2048 | 2048 | 208.80 | 2560 | 0 | 12.26 | 25109.31 | 2631.24 | 27740.55 | 365.94 | 530.02 | 4211.56 | 44.40 | 48.85 | 54.00 | 9842.38 | 16285.06 | 20408.42 | ✅ | +| c128_i2048_o256 | 128 | 2048 | 256 | 196.63 | 2560 | 0 | 13.02 | 26663.63 | 2605.54 | 29269.17 | 379.32 | 522.39 | 4205.72 | 45.73 | 50.01 | 51.66 | 9476.10 | 12538.01 | 13720.71 | ✅ | +| c128_i2048_o4096 | 128 | 2048 | 4096 | 204.98 | 2560 | 0 | 12.49 | 25578.03 | 2696.32 | 28274.35 | 369.65 | 497.47 | 4203.66 | 44.06 | 47.90 | 49.49 | 9834.14 | 15705.61 | 22444.28 | ✅ | +| c128_i2048_o512 | 128 | 2048 | 512 | 205.58 | 2560 | 0 | 12.45 | 25502.28 | 2682.24 | 28184.52 | 364.96 | 479.81 | 4198.31 | 44.32 | 48.38 | 49.46 | 9849.41 | 16273.27 | 21537.21 | ✅ | +| c128_i4096_o1024 | 128 | 4096 | 1024 | 299.56 | 2560 | 0 | 8.55 | 35004.04 | 1823.49 | 36827.53 | 670.94 | 898.57 | 8472.05 | 64.95 | 71.72 | 78.31 | 14410.80 | 23823.73 | 34763.30 | ⚠️ | +| c128_i4096_o128 | 128 | 4096 | 128 | 252.68 | 2560 | 0 | 10.13 | 41497.56 | 1283.45 | 42781.02 | 1110.40 | 1837.39 | 8464.33 | 89.25 | 99.28 | 101.99 | 12341.74 | 13909.51 | 19709.73 | ⚠️ | +| c128_i4096_o2048 | 128 | 4096 | 2048 | 299.51 | 2560 | 0 | 8.55 | 35009.54 | 1841.89 | 36851.42 | 673.64 | 903.51 | 8456.42 | 64.31 | 71.20 | 77.98 | 14467.32 | 24917.88 | 37076.16 | ⚠️ | +| c128_i4096_o256 | 128 | 4096 | 256 | 288.98 | 2560 | 0 | 8.86 | 36284.85 | 1743.96 | 38028.81 | 688.49 | 1029.32 | 8474.33 | 68.27 | 74.67 | 80.07 | 14052.64 | 18727.14 | 21844.01 | ⚠️ | +| c128_i4096_o4096 | 128 | 4096 | 4096 | 297.69 | 2560 | 0 | 8.60 | 35223.60 | 1820.84 | 37044.43 | 679.88 | 979.68 | 8471.74 | 65.18 | 72.12 | 79.04 | 14368.53 | 24055.68 | 33864.98 | ⚠️ | +| c128_i4096_o512 | 128 | 4096 | 512 | 296.90 | 2560 | 0 | 8.62 | 35317.89 | 1815.79 | 37133.68 | 666.19 | 888.77 | 8469.32 | 65.45 | 72.23 | 78.81 | 14377.25 | 24981.06 | 33674.28 | ⚠️ | +| c128_i512_o256 | 128 | 512 | 256 | 155.29 | 2560 | 0 | 16.49 | 8440.72 | 3401.04 | 11841.77 | 216.12 | 439.65 | 1351.98 | 35.17 | 38.71 | 39.83 | 7433.74 | 9800.36 | 10039.98 | ✅ | +| c16_i512_o256 | 16 | 512 | 256 | 60.82 | 320 | 0 | 5.26 | 2693.64 | 1089.67 | 3783.32 | 145.56 | 311.20 | 388.94 | 12.62 | 17.00 | 17.83 | 2748.60 | 4105.76 | 4640.23 | ✅ | +| c1_i1024_o1024 | 1 | 1024 | 1024 | 52.34 | 20 | 0 | 0.38 | 391.29 | 85.14 | 476.42 | 115.81 | 118.26 | 140.50 | 6.99 | 7.00 | 7.00 | 1666.63 | 2301.13 | 2358.69 | ✅ | +| c1_i1024_o128 | 1 | 1024 | 128 | 32.71 | 20 | 0 | 0.61 | 626.06 | 78.07 | 704.13 | 115.30 | 122.94 | 128.47 | 6.96 | 6.97 | 7.01 | 997.08 | 1004.84 | 1011.97 | ✅ | +| c1_i1024_o2048 | 1 | 1024 | 2048 | 48.50 | 20 | 0 | 0.41 | 422.30 | 83.02 | 505.31 | 115.62 | 119.50 | 126.29 | 6.99 | 7.00 | 7.01 | 1516.37 | 1700.77 | 1778.35 | ✅ | +| c1_i1024_o256 | 1 | 1024 | 256 | 48.48 | 20 | 0 | 0.41 | 422.40 | 81.26 | 503.66 | 114.81 | 120.58 | 125.23 | 6.99 | 7.01 | 7.01 | 1485.39 | 1837.55 | 1880.36 | ✅ | +| c1_i1024_o4096 | 1 | 1024 | 4096 | 51.98 | 20 | 0 | 0.38 | 393.97 | 82.72 | 476.69 | 116.51 | 126.81 | 137.83 | 6.99 | 7.00 | 7.01 | 1612.52 | 2093.77 | 2163.97 | ✅ | +| c1_i1024_o512 | 1 | 1024 | 512 | 49.68 | 20 | 0 | 0.40 | 412.23 | 78.90 | 491.13 | 132.68 | 174.48 | 175.39 | 7.00 | 7.02 | 7.02 | 1497.09 | 2052.55 | 2150.57 | ✅ | +| c1_i131072_o128 | 1 | 131072 | 128 | 257.45 | 20 | 0 | 0.08 | 10182.44 | 8.90 | 10191.34 | 6471.48 | 6574.88 | 6590.68 | 7.21 | 7.25 | 7.32 | 7290.28 | 7408.33 | 7425.83 | ⚠️ | +| c1_i131072_o256 | 1 | 131072 | 256 | 259.43 | 20 | 0 | 0.08 | 10104.51 | 13.11 | 10117.61 | 6523.80 | 6605.21 | 6683.36 | 7.25 | 7.28 | 7.30 | 7749.51 | 8208.24 | 8219.55 | ⚠️ | +| c1_i131072_o512 | 1 | 131072 | 512 | 281.03 | 20 | 0 | 0.07 | 9328.01 | 15.40 | 9343.41 | 6520.17 | 6595.29 | 6639.39 | 7.26 | 7.27 | 7.29 | 8083.13 | 9930.97 | 10147.71 | ⚠️ | +| c1_i16384_o1024 | 1 | 16384 | 1024 | 85.55 | 20 | 0 | 0.23 | 3830.19 | 66.12 | 3896.31 | 761.59 | 767.17 | 767.57 | 7.09 | 7.11 | 7.11 | 2761.83 | 4275.72 | 5529.53 | ✅ | +| c1_i16384_o128 | 1 | 16384 | 128 | 54.06 | 20 | 0 | 0.37 | 6061.09 | 46.98 | 6108.07 | 758.98 | 762.04 | 764.60 | 7.03 | 7.04 | 7.04 | 1645.31 | 1649.67 | 1651.54 | ✅ | +| c1_i16384_o2048 | 1 | 16384 | 2048 | 90.20 | 20 | 0 | 0.22 | 3632.96 | 62.71 | 3695.67 | 762.25 | 769.13 | 769.49 | 7.10 | 7.11 | 7.11 | 2762.54 | 3858.26 | 4063.08 | ✅ | +| c1_i16384_o256 | 1 | 16384 | 256 | 73.13 | 20 | 0 | 0.27 | 4480.86 | 56.19 | 4537.04 | 757.24 | 769.18 | 772.94 | 7.09 | 7.11 | 7.11 | 2205.64 | 2558.68 | 2568.89 | ✅ | +| c1_i16384_o512 | 1 | 16384 | 512 | 75.43 | 20 | 0 | 0.27 | 4344.23 | 53.96 | 4398.19 | 761.98 | 768.30 | 778.23 | 7.10 | 7.11 | 7.11 | 2198.96 | 2832.27 | 2889.46 | ✅ | +| c1_i2048_o1024 | 1 | 2048 | 1024 | 52.15 | 20 | 0 | 0.38 | 785.38 | 81.93 | 867.31 | 135.15 | 140.23 | 150.82 | 7.07 | 7.09 | 7.10 | 1639.00 | 2263.67 | 3129.84 | ✅ | +| c1_i2048_o128 | 1 | 2048 | 128 | 33.24 | 20 | 0 | 0.60 | 1232.22 | 74.97 | 1307.19 | 136.94 | 159.17 | 181.96 | 7.04 | 7.10 | 7.10 | 1006.62 | 1053.40 | 1075.84 | ✅ | +| c1_i2048_o2048 | 1 | 2048 | 2048 | 51.48 | 20 | 0 | 0.39 | 795.71 | 84.60 | 880.31 | 134.55 | 137.97 | 148.69 | 7.07 | 7.09 | 7.10 | 1667.40 | 3179.96 | 3366.92 | ✅ | +| c1_i2048_o256 | 1 | 2048 | 256 | 48.45 | 20 | 0 | 0.41 | 845.46 | 79.01 | 924.47 | 133.36 | 134.84 | 138.71 | 7.07 | 7.08 | 7.09 | 1479.65 | 1930.94 | 1931.75 | ✅ | +| c1_i2048_o4096 | 1 | 2048 | 4096 | 52.65 | 20 | 0 | 0.38 | 777.92 | 77.54 | 855.46 | 134.26 | 137.78 | 149.15 | 7.08 | 7.09 | 7.10 | 1571.48 | 2033.68 | 2095.42 | ✅ | +| c1_i2048_o512 | 1 | 2048 | 512 | 50.39 | 20 | 0 | 0.40 | 812.88 | 79.10 | 891.98 | 133.86 | 135.40 | 147.37 | 7.08 | 7.10 | 7.10 | 1537.04 | 2769.40 | 2894.64 | ✅ | +| c1_i262144_o256 | 1 | 262144 | 256 | 556.90 | 20 | 0 | 0.04 | 9414.44 | 6.27 | 9420.71 | 15234.61 | 15316.53 | 15320.27 | 7.47 | 7.55 | 7.57 | 16530.72 | 17025.05 | 17032.91 | ⚠️ | +| c1_i4096_o1024 | 1 | 4096 | 1024 | 50.77 | 20 | 0 | 0.39 | 1613.53 | 78.27 | 1691.81 | 223.98 | 228.84 | 229.98 | 7.08 | 7.09 | 7.09 | 1622.74 | 2090.63 | 2641.63 | ✅ | +| c1_i4096_o128 | 1 | 4096 | 128 | 36.18 | 20 | 0 | 0.55 | 2264.03 | 70.17 | 2334.20 | 223.37 | 225.71 | 227.21 | 7.03 | 7.04 | 7.07 | 1108.62 | 1117.89 | 1119.93 | ✅ | +| c1_i4096_o2048 | 1 | 4096 | 2048 | 54.09 | 20 | 0 | 0.37 | 1514.64 | 78.34 | 1592.98 | 223.30 | 226.85 | 227.97 | 7.08 | 7.09 | 7.09 | 1715.15 | 2736.41 | 2965.24 | ✅ | +| c1_i4096_o256 | 1 | 4096 | 256 | 53.93 | 20 | 0 | 0.37 | 1519.00 | 77.17 | 1596.18 | 224.11 | 227.58 | 229.35 | 7.07 | 7.09 | 7.09 | 1687.85 | 2022.93 | 2023.17 | ✅ | +| c1_i4096_o4096 | 1 | 4096 | 4096 | 50.60 | 20 | 0 | 0.40 | 1618.96 | 72.98 | 1691.94 | 223.43 | 228.43 | 229.73 | 7.08 | 7.09 | 7.09 | 1522.86 | 2146.74 | 2282.49 | ✅ | +| c1_i4096_o512 | 1 | 4096 | 512 | 59.75 | 20 | 0 | 0.33 | 1371.10 | 81.14 | 1452.25 | 224.35 | 229.07 | 230.84 | 7.07 | 7.08 | 7.08 | 1931.53 | 2977.28 | 3648.36 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 53.26 | 20 | 0 | 0.38 | 192.26 | 80.81 | 273.07 | 167.28 | 203.13 | 205.33 | 6.99 | 7.02 | 7.03 | 1663.97 | 1951.39 | 1953.26 | ✅ | +| c1_i524288_o128 | 1 | 524288 | 128 | 1513.75 | 20 | 0 | 0.01 | 6927.01 | 1.20 | 6928.22 | 40602.84 | 41121.70 | 41219.49 | 7.78 | 7.92 | 7.93 | 41303.57 | 41857.14 | 41958.45 | ⚠️ | +| c1_i65536_o1024 | 1 | 65536 | 1024 | 152.81 | 20 | 0 | 0.13 | 8577.64 | 26.28 | 8603.92 | 2984.49 | 2994.56 | 3006.30 | 7.18 | 7.18 | 7.19 | 4418.27 | 5282.47 | 5593.15 | ✅ | +| c1_i65536_o128 | 1 | 65536 | 128 | 134.86 | 20 | 0 | 0.15 | 9718.82 | 17.89 | 9736.71 | 2996.65 | 3058.30 | 3078.26 | 7.11 | 7.16 | 7.16 | 3847.83 | 3911.31 | 3932.26 | ⚠️ | +| c1_i65536_o256 | 1 | 65536 | 256 | 153.56 | 20 | 0 | 0.13 | 8535.68 | 25.96 | 8561.64 | 2986.14 | 2994.62 | 3008.90 | 7.16 | 7.18 | 7.21 | 4407.24 | 4762.51 | 4778.90 | ✅ | +| c1_i65536_o512 | 1 | 65536 | 512 | 157.16 | 20 | 0 | 0.13 | 8340.12 | 27.70 | 8367.82 | 2986.06 | 2996.34 | 3023.15 | 7.18 | 7.19 | 7.20 | 4541.31 | 6107.66 | 6492.71 | ✅ | +| c2_i512_o256 | 2 | 512 | 256 | 44.94 | 40 | 0 | 0.89 | 455.73 | 188.32 | 644.06 | 123.41 | 177.91 | 187.26 | 7.06 | 7.48 | 7.68 | 1611.58 | 2020.21 | 2050.24 | ✅ | +| c32_i1024_o1024 | 32 | 1024 | 1024 | 78.93 | 640 | 0 | 8.11 | 8303.55 | 1684.20 | 9987.75 | 154.93 | 217.27 | 815.30 | 16.71 | 18.61 | 18.94 | 3611.27 | 4919.69 | 5510.54 | ✅ | +| c32_i1024_o128 | 32 | 1024 | 128 | 46.47 | 640 | 0 | 13.77 | 14103.72 | 1762.90 | 15866.62 | 408.25 | 714.24 | 757.98 | 13.41 | 16.99 | 17.83 | 2110.85 | 2431.17 | 2650.42 | ✅ | +| c32_i1024_o2048 | 32 | 1024 | 2048 | 81.00 | 640 | 0 | 7.90 | 8090.92 | 1622.21 | 9713.13 | 162.95 | 334.37 | 753.35 | 17.45 | 23.66 | 25.29 | 3730.41 | 5542.76 | 6244.95 | ✅ | +| c32_i1024_o256 | 32 | 1024 | 256 | 76.79 | 640 | 0 | 8.33 | 8534.95 | 1672.11 | 10207.06 | 154.72 | 212.02 | 753.62 | 16.97 | 19.11 | 20.00 | 3544.98 | 4746.50 | 4887.24 | ✅ | +| c32_i1024_o4096 | 32 | 1024 | 4096 | 77.85 | 640 | 0 | 8.22 | 8418.72 | 1664.78 | 10083.50 | 154.28 | 214.19 | 756.53 | 16.85 | 18.82 | 19.49 | 3549.62 | 4954.17 | 5479.06 | ✅ | +| c32_i1024_o512 | 32 | 1024 | 512 | 77.73 | 640 | 0 | 8.23 | 8431.17 | 1700.51 | 10131.68 | 155.17 | 216.90 | 751.85 | 16.62 | 18.26 | 18.67 | 3574.12 | 4831.87 | 5478.82 | ✅ | +| c32_i16384_o1024 | 32 | 16384 | 1024 | 277.96 | 640 | 0 | 2.30 | 37723.90 | 563.76 | 38287.65 | 1266.27 | 1984.25 | 8777.02 | 48.75 | 60.64 | 65.85 | 13073.64 | 26643.75 | 35597.74 | ✅ | +| c32_i16384_o128 | 32 | 16384 | 128 | 240.94 | 640 | 0 | 2.66 | 43520.72 | 338.90 | 43859.62 | 3738.21 | 4848.25 | 8794.30 | 61.61 | 81.93 | 82.76 | 11539.01 | 13150.69 | 16925.01 | ❌ | +| c32_i16384_o2048 | 32 | 16384 | 2048 | 277.48 | 640 | 0 | 2.31 | 37789.10 | 576.94 | 38366.04 | 1268.99 | 2250.82 | 8796.37 | 47.57 | 58.28 | 64.83 | 13129.88 | 24681.93 | 38211.25 | ✅ | +| c32_i16384_o256 | 32 | 16384 | 256 | 264.70 | 640 | 0 | 2.42 | 39614.37 | 508.66 | 40123.04 | 1298.03 | 1994.66 | 8761.72 | 54.06 | 64.54 | 70.67 | 12625.00 | 16418.68 | 19908.81 | ⚠️ | +| c32_i16384_o512 | 32 | 16384 | 512 | 275.93 | 640 | 0 | 2.32 | 38001.51 | 561.14 | 38562.65 | 1268.67 | 1990.86 | 8787.03 | 48.89 | 60.15 | 66.01 | 13034.17 | 25370.34 | 27540.55 | ✅ | +| c32_i2048_o1024 | 32 | 2048 | 1024 | 87.95 | 640 | 0 | 7.28 | 14903.68 | 1574.85 | 16478.54 | 200.89 | 270.09 | 1352.10 | 17.77 | 19.91 | 20.51 | 4023.46 | 6506.86 | 8623.71 | ✅ | +| c32_i2048_o128 | 32 | 2048 | 128 | 59.24 | 640 | 0 | 10.80 | 22125.82 | 1375.94 | 23501.76 | 367.53 | 672.56 | 1352.00 | 18.71 | 21.55 | 22.09 | 2732.47 | 3094.92 | 3404.73 | ✅ | +| c32_i2048_o2048 | 32 | 2048 | 2048 | 89.13 | 640 | 0 | 7.18 | 14705.49 | 1556.10 | 16261.59 | 199.91 | 291.10 | 1361.13 | 17.87 | 20.28 | 20.98 | 4056.53 | 6825.74 | 9339.78 | ✅ | +| c32_i2048_o256 | 32 | 2048 | 256 | 83.40 | 640 | 0 | 7.67 | 15715.67 | 1544.12 | 17259.80 | 202.63 | 380.57 | 1355.98 | 18.19 | 20.18 | 20.73 | 3838.39 | 5110.33 | 5339.24 | ✅ | +| c32_i2048_o4096 | 32 | 2048 | 4096 | 88.69 | 640 | 0 | 7.22 | 14778.01 | 1520.45 | 16298.46 | 199.19 | 288.11 | 1360.76 | 18.06 | 20.41 | 20.84 | 3982.98 | 6383.30 | 9405.39 | ✅ | +| c32_i2048_o512 | 32 | 2048 | 512 | 90.84 | 640 | 0 | 7.05 | 14428.41 | 1530.32 | 15958.73 | 197.17 | 253.30 | 1349.46 | 17.88 | 20.25 | 21.02 | 4060.40 | 6700.48 | 9080.73 | ✅ | +| c32_i4096_o1024 | 32 | 4096 | 1024 | 113.81 | 640 | 0 | 5.62 | 23032.89 | 1212.69 | 24245.58 | 339.67 | 542.37 | 2422.62 | 22.72 | 26.10 | 27.21 | 5215.72 | 8904.34 | 13457.62 | ✅ | +| c32_i4096_o128 | 32 | 4096 | 128 | 85.66 | 640 | 0 | 7.47 | 30604.61 | 950.87 | 31555.48 | 556.18 | 1180.23 | 2449.13 | 27.29 | 31.52 | 34.82 | 4002.80 | 4613.66 | 5507.05 | ✅ | +| c32_i4096_o2048 | 32 | 4096 | 2048 | 114.27 | 640 | 0 | 5.60 | 22939.90 | 1189.49 | 24129.39 | 341.11 | 550.60 | 2435.46 | 22.78 | 26.02 | 27.73 | 5165.78 | 9243.12 | 13018.90 | ✅ | +| c32_i4096_o256 | 32 | 4096 | 256 | 106.21 | 640 | 0 | 6.03 | 24681.91 | 1178.61 | 25860.52 | 350.79 | 564.37 | 2447.99 | 23.55 | 26.71 | 28.32 | 4937.54 | 6734.16 | 7405.49 | ✅ | +| c32_i4096_o4096 | 32 | 4096 | 4096 | 114.94 | 640 | 0 | 5.57 | 22806.49 | 1192.22 | 23998.71 | 341.86 | 550.55 | 2448.39 | 22.83 | 26.16 | 27.23 | 5203.63 | 8545.93 | 12625.46 | ✅ | +| c32_i4096_o512 | 32 | 4096 | 512 | 112.20 | 640 | 0 | 5.70 | 23364.93 | 1200.40 | 24565.33 | 340.20 | 560.25 | 2385.06 | 22.99 | 26.29 | 27.73 | 5150.64 | 8529.09 | 12164.94 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 78.27 | 640 | 0 | 8.18 | 4186.51 | 1690.88 | 5877.39 | 149.00 | 289.93 | 473.49 | 16.81 | 19.27 | 22.36 | 3610.39 | 4786.53 | 5755.87 | ✅ | +| c32_i65536_o1024 | 32 | 65536 | 1024 | 266.52 | 160 | 0 | 0.60 | 39343.10 | 131.31 | 39474.42 | 8656.40 | 33860.81 | 43057.06 | 171.14 | 244.27 | 281.95 | 46303.99 | 103438.83 | 127750.75 | ❌ | +| c4_i512_o256 | 4 | 512 | 256 | 43.40 | 80 | 0 | 1.84 | 943.85 | 379.18 | 1323.03 | 118.07 | 125.54 | 241.46 | 7.83 | 8.29 | 8.68 | 1720.27 | 2165.16 | 2253.32 | ✅ | +| c64_i512_o256 | 64 | 512 | 256 | 109.33 | 1280 | 0 | 11.71 | 5994.26 | 2402.36 | 8396.62 | 171.45 | 299.06 | 771.51 | 24.39 | 26.72 | 27.58 | 5151.60 | 6720.29 | 6878.42 | ✅ | +| c8_i1024_o1024 | 8 | 1024 | 1024 | 47.80 | 160 | 0 | 3.35 | 3427.67 | 696.10 | 4123.77 | 129.12 | 192.22 | 281.04 | 9.27 | 9.95 | 10.24 | 2052.01 | 2788.77 | 3052.14 | ✅ | +| c8_i1024_o128 | 8 | 1024 | 128 | 31.03 | 160 | 0 | 5.16 | 5279.38 | 659.92 | 5939.30 | 187.55 | 269.21 | 292.69 | 8.88 | 9.90 | 9.97 | 1315.49 | 1387.68 | 1458.94 | ✅ | +| c8_i1024_o2048 | 8 | 1024 | 2048 | 47.63 | 160 | 0 | 3.36 | 3439.84 | 684.76 | 4124.59 | 139.64 | 191.21 | 283.85 | 9.49 | 10.93 | 11.77 | 2064.28 | 2867.60 | 3177.60 | ✅ | +| c8_i1024_o256 | 8 | 1024 | 256 | 47.68 | 160 | 0 | 3.36 | 3436.34 | 684.94 | 4121.28 | 134.22 | 195.34 | 290.76 | 9.44 | 10.83 | 11.66 | 2050.72 | 2669.20 | 2870.35 | ✅ | +| c8_i1024_o4096 | 8 | 1024 | 4096 | 48.02 | 160 | 0 | 3.33 | 3411.81 | 683.42 | 4095.23 | 132.59 | 194.92 | 285.39 | 9.39 | 10.45 | 11.05 | 2050.12 | 2647.51 | 2950.32 | ✅ | +| c8_i1024_o512 | 8 | 1024 | 512 | 48.89 | 160 | 0 | 3.27 | 3351.35 | 699.85 | 4051.20 | 128.43 | 130.82 | 286.17 | 9.25 | 9.96 | 10.23 | 2097.31 | 2846.33 | 3059.68 | ✅ | +| c8_i131072_o512 | 8 | 131072 | 512 | 593.71 | 160 | 0 | 0.27 | 35322.95 | 49.97 | 35372.92 | 8837.26 | 12856.65 | 21274.84 | 95.98 | 174.74 | 250.31 | 26223.81 | 43791.96 | 52427.10 | ❌ | +| c8_i16384_o1024 | 8 | 16384 | 1024 | 117.03 | 160 | 0 | 1.37 | 22399.35 | 381.47 | 22780.82 | 880.55 | 1325.49 | 2519.24 | 15.14 | 18.88 | 22.89 | 5043.96 | 9664.45 | 11942.89 | ✅ | +| c8_i16384_o128 | 8 | 16384 | 128 | 83.84 | 160 | 0 | 1.91 | 31265.56 | 241.22 | 31506.78 | 1766.89 | 2637.47 | 2676.54 | 14.95 | 23.25 | 26.08 | 3640.27 | 3713.62 | 5332.84 | ✅ | +| c8_i16384_o2048 | 8 | 16384 | 2048 | 111.65 | 160 | 0 | 1.43 | 23479.38 | 353.00 | 23832.38 | 858.33 | 1105.07 | 2515.80 | 15.80 | 19.89 | 21.08 | 4778.32 | 8910.12 | 13501.67 | ✅ | +| c8_i16384_o256 | 8 | 16384 | 256 | 101.77 | 160 | 0 | 1.57 | 25759.71 | 329.17 | 26088.88 | 867.77 | 1267.33 | 2511.92 | 17.06 | 21.93 | 23.13 | 4427.86 | 5958.68 | 6667.56 | ✅ | +| c8_i16384_o512 | 8 | 16384 | 512 | 109.64 | 160 | 0 | 1.46 | 23908.84 | 355.69 | 24264.53 | 908.35 | 1361.76 | 2515.33 | 15.82 | 20.35 | 22.15 | 4755.66 | 8684.67 | 10122.99 | ✅ | +| c8_i2048_o1024 | 8 | 2048 | 1024 | 53.11 | 160 | 0 | 3.01 | 6170.36 | 677.84 | 6848.20 | 150.15 | 153.04 | 410.55 | 9.49 | 10.24 | 10.65 | 2286.09 | 3870.04 | 5178.54 | ✅ | +| c8_i2048_o128 | 8 | 2048 | 128 | 33.35 | 160 | 0 | 4.80 | 9826.22 | 612.40 | 10438.62 | 228.93 | 388.77 | 404.46 | 9.33 | 10.33 | 11.02 | 1411.23 | 1530.95 | 1595.88 | ✅ | +| c8_i2048_o2048 | 8 | 2048 | 2048 | 51.12 | 160 | 0 | 3.13 | 6409.90 | 672.70 | 7082.60 | 150.70 | 165.44 | 418.00 | 9.56 | 10.25 | 10.68 | 2202.49 | 3406.00 | 4277.62 | ✅ | +| c8_i2048_o256 | 8 | 2048 | 256 | 48.03 | 160 | 0 | 3.33 | 6822.11 | 658.85 | 7480.96 | 148.57 | 147.18 | 403.31 | 9.66 | 10.48 | 10.98 | 2051.77 | 2715.99 | 2745.12 | ✅ | +| c8_i2048_o4096 | 8 | 2048 | 4096 | 53.12 | 160 | 0 | 3.01 | 6168.50 | 661.05 | 6829.55 | 150.99 | 219.64 | 421.18 | 9.58 | 10.44 | 10.58 | 2250.24 | 3788.73 | 4649.51 | ✅ | +| c8_i2048_o512 | 8 | 2048 | 512 | 50.97 | 160 | 0 | 3.14 | 6428.69 | 669.04 | 7097.73 | 148.41 | 147.71 | 403.41 | 9.60 | 10.48 | 11.08 | 2190.12 | 3392.37 | 4269.13 | ✅ | +| c8_i262144_o128 | 8 | 262144 | 128 | 1333.56 | 160 | 0 | 0.12 | 31451.94 | 12.61 | 31464.55 | 36128.45 | 57997.25 | 58592.27 | 215.95 | 392.14 | 446.94 | 59143.65 | 61777.27 | 74358.75 | ❌ | +| c8_i4096_o1024 | 8 | 4096 | 1024 | 60.18 | 160 | 0 | 2.66 | 10890.64 | 578.38 | 11469.02 | 253.10 | 364.29 | 731.93 | 10.57 | 11.89 | 12.17 | 2550.26 | 4642.49 | 7535.89 | ✅ | +| c8_i4096_o128 | 8 | 4096 | 128 | 40.35 | 160 | 0 | 3.96 | 16240.24 | 506.02 | 16746.26 | 376.51 | 695.40 | 727.26 | 10.62 | 11.99 | 13.73 | 1721.35 | 1910.10 | 2181.08 | ✅ | +| c8_i4096_o2048 | 8 | 4096 | 2048 | 57.14 | 160 | 0 | 2.80 | 11470.28 | 593.22 | 12063.50 | 254.35 | 361.27 | 732.14 | 10.52 | 11.68 | 12.36 | 2478.80 | 3843.55 | 5407.45 | ✅ | +| c8_i4096_o256 | 8 | 4096 | 256 | 54.73 | 160 | 0 | 2.92 | 11973.53 | 575.24 | 12548.76 | 252.76 | 366.29 | 732.21 | 10.63 | 12.02 | 12.59 | 2342.56 | 3122.99 | 3247.98 | ✅ | +| c8_i4096_o4096 | 8 | 4096 | 4096 | 61.97 | 160 | 0 | 2.58 | 10574.93 | 602.60 | 11177.53 | 251.62 | 276.31 | 733.83 | 10.48 | 11.81 | 12.23 | 2715.50 | 3797.26 | 5487.12 | ✅ | +| c8_i4096_o512 | 8 | 4096 | 512 | 57.59 | 160 | 0 | 2.78 | 11379.47 | 586.75 | 11966.22 | 255.02 | 376.23 | 733.95 | 10.54 | 11.70 | 12.17 | 2482.90 | 4048.32 | 5636.75 | ✅ | +| c8_i512_o256 | 8 | 512 | 256 | 48.34 | 160 | 0 | 3.31 | 1694.53 | 687.00 | 2381.52 | 125.68 | 194.69 | 258.22 | 9.43 | 10.43 | 10.95 | 2075.15 | 2669.68 | 2871.40 | ✅ | +| c8_i65536_o1024 | 8 | 65536 | 1024 | 298.66 | 160 | 0 | 0.54 | 35109.32 | 107.56 | 35216.88 | 3847.52 | 5884.53 | 9844.52 | 46.16 | 82.22 | 101.31 | 13001.39 | 21336.69 | 30533.51 | ⚠️ | +| c8_i65536_o128 | 8 | 65536 | 128 | 274.11 | 160 | 0 | 0.58 | 38254.29 | 70.30 | 38324.59 | 7223.66 | 11101.75 | 11262.28 | 40.50 | 72.54 | 72.63 | 12089.84 | 12347.21 | 12881.32 | ⚠️ | +| c8_i65536_o256 | 8 | 65536 | 256 | 291.01 | 160 | 0 | 0.55 | 36032.88 | 98.87 | 36131.75 | 3804.89 | 5718.15 | 9852.36 | 50.92 | 78.36 | 96.49 | 12843.24 | 18584.33 | 21293.29 | ❌ | +| c8_i65536_o512 | 8 | 65536 | 512 | 297.04 | 160 | 0 | 0.54 | 35301.20 | 111.90 | 35413.10 | 3902.29 | 6055.87 | 9835.60 | 44.23 | 73.93 | 83.82 | 13139.03 | 22983.14 | 31590.46 | ⚠️ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json new file mode 100644 index 0000000..4f3bbb4 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/results.json @@ -0,0 +1,6429 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp4_custom_bench", + "run_id": "20260711-133908", + "timestamp": "2026-07-11T13:39:08+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "def1355", + "git_dirty": "dirty", + "description": "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py" + }, + "config": { + "tp": 4, + "num_services": 2, + "ports": "30005,30006", + "gpu_groups": "0,1,2,3|4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "max_num_seqs": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "2 512 256 40", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [ + { + "name": "c128_i1024_o1024", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 172.39323008002248, + "request_throughput": 14.849771065903717, + "input_token_throughput": 15206.165571485406, + "output_token_throughput": 3100.5973944097605, + "total_token_throughput": 18306.762965895166, + "total_input_tokens": 2621440, + "total_output_tokens": 534522, + "e2e_ms": { + "mean": 8264.996455389724, + "p50": 7789.678987493971, + "p90": 11756.317305739502, + "p95": 13967.23252292723, + "p99": 17536.458535679838 + }, + "ttft_ms": { + "mean": 267.1602980701664, + "p50": 194.55134301097132, + "p90": 328.0344645609148, + "p95": 478.8006908784156, + "p99": 2297.698974920786 + }, + "tpot_ms": { + "mean": 38.53003130054492, + "p50": 36.742621067679, + "p90": 59.41429369279392, + "p95": 66.1565177263384, + "p99": 70.07324518675013 + }, + "itl_ms": { + "mean": 38.33531172431711, + "p50": 36.54410010322313, + "p90": 59.16513506822693, + "p95": 65.93954884337958, + "p99": 69.76542250785654 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 98.83996513002785, + "request_throughput": 25.900454301377177, + "input_token_throughput": 26522.06520461023, + "output_token_throughput": 3314.9141601675883, + "total_token_throughput": 29836.979364777817, + "total_input_tokens": 2621440, + "total_output_tokens": 327646, + "e2e_ms": { + "mean": 4730.889936950962, + "p50": 4738.875214505242, + "p90": 5273.3935622207355, + "p95": 5530.639346732642, + "p99": 6122.148045647769 + }, + "ttft_ms": { + "mean": 900.3881096975647, + "p50": 878.4408235223964, + "p90": 1408.8249116961379, + "p95": 1590.8718543098075, + "p99": 2307.9329636477605 + }, + "tpot_ms": { + "mean": 30.164361933300313, + "p50": 30.757310448716737, + "p90": 35.303492756518274, + "p95": 35.98104450527641, + "p99": 37.085646242618296 + }, + "itl_ms": { + "mean": 30.16279034305132, + "p50": 30.757079614091694, + "p90": 35.303211258999326, + "p95": 35.9806825589264, + "p99": 37.08532446034263 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_128.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.76455012900988, + "request_throughput": 15.825469782831641, + "input_token_throughput": 16205.2810576196, + "output_token_throughput": 3297.7435388319536, + "total_token_throughput": 19503.024596451552, + "total_input_tokens": 2621440, + "total_output_tokens": 533458, + "e2e_ms": { + "mean": 7732.00934715162, + "p50": 7706.126845994731, + "p90": 9936.003248777706, + "p95": 10616.589111639772, + "p99": 11870.42382700427 + }, + "ttft_ms": { + "mean": 256.6061660457535, + "p50": 183.6360429879278, + "p90": 278.1436800782103, + "p95": 340.86536000250027, + "p99": 2315.9884250303726 + }, + "tpot_ms": { + "mean": 36.0673758854372, + "p50": 36.71776657082684, + "p90": 38.754329601805686, + "p95": 39.269018607056736, + "p99": 40.296127494022976 + }, + "itl_ms": { + "mean": 35.875113844333285, + "p50": 36.535086806523225, + "p90": 38.54275344150884, + "p95": 39.06941631082252, + "p99": 40.037952805839154 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.15555513004074, + "request_throughput": 15.885273070073616, + "input_token_throughput": 16266.519623755383, + "output_token_throughput": 3250.7163626923966, + "total_token_throughput": 19517.235986447777, + "total_input_tokens": 2621440, + "total_output_tokens": 523871, + "e2e_ms": { + "mean": 7727.166688055763, + "p50": 7802.593892032746, + "p90": 9742.186820518691, + "p95": 10133.950301926234, + "p99": 11225.207591401988 + }, + "ttft_ms": { + "mean": 259.6949904462008, + "p50": 185.23747954168357, + "p90": 282.4472953216173, + "p95": 356.24627698853345, + "p99": 2333.3000153169146 + }, + "tpot_ms": { + "mean": 36.67608677964262, + "p50": 37.08397564475155, + "p90": 39.790116985757045, + "p95": 40.80520354344157, + "p99": 51.98653523430576 + }, + "itl_ms": { + "mean": 36.49728808087508, + "p50": 36.881504635488966, + "p90": 39.64186447351697, + "p95": 40.63193390640941, + "p99": 51.59359547846483 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_256.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 163.1641573330271, + "request_throughput": 15.689720351847237, + "input_token_throughput": 16066.27364029157, + "output_token_throughput": 3281.0330942188916, + "total_token_throughput": 19347.30673451046, + "total_input_tokens": 2621440, + "total_output_tokens": 535347, + "e2e_ms": { + "mean": 7768.0705221036305, + "p50": 7779.976884485222, + "p90": 9995.535276620649, + "p95": 10730.881073744968, + "p99": 12234.15256866486 + }, + "ttft_ms": { + "mean": 257.28251816292413, + "p50": 183.7318860052619, + "p90": 284.0460309351329, + "p95": 422.5844055210541, + "p99": 2303.9169573405525 + }, + "tpot_ms": { + "mean": 36.08731376501835, + "p50": 36.74589582311257, + "p90": 38.635312321757525, + "p95": 39.20972573694193, + "p99": 40.34653792474833 + }, + "itl_ms": { + "mean": 35.90677394448389, + "p50": 36.55651061150313, + "p90": 38.41832325335143, + "p95": 39.02278168168324, + "p99": 40.18393781864987 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 161.4934899629443, + "request_throughput": 15.852032181528854, + "input_token_throughput": 16232.480953885546, + "output_token_throughput": 3275.9153333140016, + "total_token_throughput": 19508.39628719955, + "total_input_tokens": 2621440, + "total_output_tokens": 529039, + "e2e_ms": { + "mean": 7716.2186734600255, + "p50": 7756.274058541749, + "p90": 9902.339230419602, + "p95": 10581.986936813337, + "p99": 11832.60301870119 + }, + "ttft_ms": { + "mean": 255.16095905081784, + "p50": 183.0740385048557, + "p90": 275.3751680254936, + "p95": 334.6741058048795, + "p99": 2297.212610503774 + }, + "tpot_ms": { + "mean": 36.27411559738953, + "p50": 36.90445407913109, + "p90": 38.89796617059789, + "p95": 39.38925744135109, + "p99": 40.574045828890846 + }, + "itl_ms": { + "mean": 36.088637286494745, + "p50": 36.715503331749815, + "p90": 38.668252778799605, + "p95": 39.183733808278504, + "p99": 40.430493946190595 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_1024_512.jsonl" + }, + { + "name": "c128_i16384_o1024", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 919.2520269450033, + "request_throughput": 2.7848728367864224, + "input_token_throughput": 45627.356557908744, + "output_token_throughput": 697.3343340132238, + "total_token_throughput": 46324.690891921964, + "total_input_tokens": 41943040, + "total_output_tokens": 641026, + "e2e_ms": { + "mean": 45001.22985192386, + "p50": 38537.9672049894, + "p90": 73411.154619226, + "p95": 90438.25863343383, + "p99": 123707.69347336027 + }, + "ttft_ms": { + "mean": 2625.9279194736337, + "p50": 1610.5919524852652, + "p90": 2643.259297683834, + "p95": 3630.1065946550807, + "p99": 33634.44761375713 + }, + "tpot_ms": { + "mean": 171.29838262030282, + "p50": 174.10042585931788, + "p90": 191.857317470299, + "p95": 196.6860248351331, + "p99": 207.7062569780722 + }, + "itl_ms": { + "mean": 170.48918588362815, + "p50": 173.18544062971623, + "p90": 191.07016889404835, + "p95": 195.60057769858426, + "p99": 207.12842061635322 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_1024.jsonl" + }, + { + "name": "c128_i16384_o128", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 859.3701289989986, + "request_throughput": 2.9789259756816415, + "input_token_throughput": 48806.723185568015, + "output_token_throughput": 380.9545956424342, + "total_token_throughput": 49187.67778121045, + "total_input_tokens": 41943040, + "total_output_tokens": 327381, + "e2e_ms": { + "mean": 42403.99054401498, + "p50": 42587.17669401085, + "p90": 43794.49558101478, + "p95": 44239.059685051325, + "p99": 73426.73798753004 + }, + "ttft_ms": { + "mean": 2833.193887487778, + "p50": 1653.9359024900477, + "p90": 2892.610295308985, + "p95": 3131.2537917227023, + "p99": 33643.84073065702 + }, + "tpot_ms": { + "mean": 311.87951799655093, + "p50": 319.0447975710441, + "p90": 327.9145556648709, + "p95": 328.0359147038855, + "p99": 328.1002778994097 + }, + "itl_ms": { + "mean": 311.8659649233483, + "p50": 319.0443645553172, + "p90": 327.91121197616, + "p95": 328.03315464503453, + "p99": 328.09282922379373 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_128.jsonl" + }, + { + "name": "c128_i16384_o2048", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 918.609304343001, + "request_throughput": 2.7868213264298896, + "input_token_throughput": 45659.28061222731, + "output_token_throughput": 701.9861403006408, + "total_token_throughput": 46361.26675252795, + "total_input_tokens": 41943040, + "total_output_tokens": 644851, + "e2e_ms": { + "mean": 45030.70892128417, + "p50": 38617.496168997604, + "p90": 73778.13227887964, + "p95": 92401.43465625009, + "p99": 124284.89116791805 + }, + "ttft_ms": { + "mean": 2601.3592253009847, + "p50": 1578.1983239867259, + "p90": 2635.9606526442803, + "p95": 3607.430891436526, + "p99": 33624.65822858319 + }, + "tpot_ms": { + "mean": 170.03327446799767, + "p50": 173.23042250405052, + "p90": 190.40878788659077, + "p95": 194.99764928126098, + "p99": 208.7899013259607 + }, + "itl_ms": { + "mean": 169.23891149830678, + "p50": 172.44599117557388, + "p90": 189.46610969006963, + "p95": 194.15233367186605, + "p99": 206.90664604638008 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_2048.jsonl" + }, + { + "name": "c128_i16384_o256", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 897.621502821974, + "request_throughput": 2.8519815890682008, + "input_token_throughput": 46726.8663552934, + "output_token_throughput": 602.4231798146295, + "total_token_throughput": 47329.28953510804, + "total_input_tokens": 41943040, + "total_output_tokens": 540748, + "e2e_ms": { + "mean": 44219.84462311193, + "p50": 44721.17852998781, + "p90": 55222.73939058068, + "p95": 57755.15577839106, + "p99": 76587.71378947243 + }, + "ttft_ms": { + "mean": 2804.707829081053, + "p50": 1716.4993534679525, + "p90": 3034.1674943047105, + "p95": 4596.590572808054, + "p99": 33617.3651075537 + }, + "tpot_ms": { + "mean": 197.05998115773372, + "p50": 201.09523076543266, + "p90": 218.96908414091612, + "p95": 224.66402557030736, + "p99": 236.73202214477985 + }, + "itl_ms": { + "mean": 196.31561704005404, + "p50": 200.32954269678584, + "p90": 217.82939194011792, + "p95": 223.82267243435942, + "p99": 236.3526946892711 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_256.jsonl" + }, + { + "name": "c128_i16384_o512", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 915.1275745470193, + "request_throughput": 2.7974241747301507, + "input_token_throughput": 45832.99767877879, + "output_token_throughput": 684.5581068957425, + "total_token_throughput": 46517.55578567453, + "total_input_tokens": 41943040, + "total_output_tokens": 626458, + "e2e_ms": { + "mean": 44914.537590626765, + "p50": 39495.58882598649, + "p90": 74920.82639939617, + "p95": 88267.88074479846, + "p99": 100313.26691627383 + }, + "ttft_ms": { + "mean": 2646.0264690055737, + "p50": 1644.704559526872, + "p90": 2713.0657975503705, + "p95": 3737.5678659940886, + "p99": 33718.316934909584 + }, + "tpot_ms": { + "mean": 174.27917813481235, + "p50": 176.57178472857964, + "p90": 196.3337238559724, + "p95": 203.88145810190872, + "p99": 224.1069659303989 + }, + "itl_ms": { + "mean": 173.4964124988034, + "p50": 175.92428582230116, + "p90": 195.45350120960254, + "p95": 203.05172497723345, + "p99": 222.28400312181145 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_16384_512.jsonl" + }, + { + "name": "c128_i2048_o1024", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 204.29565773101058, + "request_throughput": 12.530858601853733, + "input_token_throughput": 25663.198416596446, + "output_token_throughput": 2671.3930489828444, + "total_token_throughput": 28334.59146557929, + "total_input_tokens": 5242880, + "total_output_tokens": 545754, + "e2e_ms": { + "mean": 9762.615562429983, + "p50": 9157.230406504823, + "p90": 13910.17175068846, + "p95": 15935.838111440538, + "p99": 21541.334415340087 + }, + "ttft_ms": { + "mean": 372.843854514349, + "p50": 242.41412596893497, + "p90": 392.64738988131285, + "p95": 493.00015218323176, + "p99": 4208.031118083745 + }, + "tpot_ms": { + "mean": 44.23840402224793, + "p50": 44.99556358921123, + "p90": 47.38613093340097, + "p95": 48.11152710137389, + "p99": 50.5311058701371 + }, + "itl_ms": { + "mean": 44.00457448991474, + "p50": 44.76411662840158, + "p90": 47.14332725574256, + "p95": 47.914761956692956, + "p99": 50.14406648849003 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_1024.jsonl" + }, + { + "name": "c128_i2048_o128", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 153.28961618000176, + "request_throughput": 16.700413660073988, + "input_token_throughput": 34202.44717583153, + "output_token_throughput": 2125.5386249868307, + "total_token_throughput": 36327.98580081836, + "total_input_tokens": 5242880, + "total_output_tokens": 325823, + "e2e_ms": { + "mean": 7426.542717551774, + "p50": 7519.444196979748, + "p90": 7931.7692122771405, + "p95": 8226.8541852216, + "p99": 10576.912148120464 + }, + "ttft_ms": { + "mean": 791.1919670015322, + "p50": 736.483736516675, + "p90": 1221.6598160099238, + "p95": 1283.0562591931086, + "p99": 4199.415099230828 + }, + "tpot_ms": { + "mean": 52.51428230122135, + "p50": 53.82101512220725, + "p90": 57.72905067094294, + "p95": 58.15436926695123, + "p99": 59.97293118720664 + }, + "itl_ms": { + "mean": 52.48190387140439, + "p50": 53.81724940943825, + "p90": 57.726806252012324, + "p95": 58.15410469809921, + "p99": 59.97246126387649 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_128.jsonl" + }, + { + "name": "c128_i2048_o2048", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.80225611501373, + "request_throughput": 12.260403922981968, + "input_token_throughput": 25109.30723426707, + "output_token_throughput": 2631.24072614125, + "total_token_throughput": 27740.54796040832, + "total_input_tokens": 5242880, + "total_output_tokens": 549409, + "e2e_ms": { + "mean": 9842.37848147145, + "p50": 9232.52340304316, + "p90": 13945.226166764041, + "p95": 16285.06117714278, + "p99": 20408.415713311373 + }, + "ttft_ms": { + "mean": 365.94149188463234, + "p50": 239.27259998163208, + "p90": 353.6969173233956, + "p95": 530.0150224182293, + "p99": 4211.561568837496 + }, + "tpot_ms": { + "mean": 44.40326790287457, + "p50": 44.98828430131604, + "p90": 47.74656392552758, + "p95": 48.85120712544617, + "p99": 53.997538806530045 + }, + "itl_ms": { + "mean": 44.172192415332475, + "p50": 44.73914634153712, + "p90": 47.505158283643, + "p95": 48.59071167813023, + "p99": 53.70061425829482 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_2048.jsonl" + }, + { + "name": "c128_i2048_o256", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 196.63041066098958, + "request_throughput": 13.019349303062256, + "input_token_throughput": 26663.6273726715, + "output_token_throughput": 2605.5379647419063, + "total_token_throughput": 29269.165337413408, + "total_input_tokens": 5242880, + "total_output_tokens": 512328, + "e2e_ms": { + "mean": 9476.103467166013, + "p50": 9467.947298486251, + "p90": 12215.088280598866, + "p95": 12538.011490146164, + "p99": 13720.708791241632 + }, + "ttft_ms": { + "mean": 379.32366461509446, + "p50": 246.58072050078772, + "p90": 404.3491132790223, + "p95": 522.392269448028, + "p99": 4205.7175672921585 + }, + "tpot_ms": { + "mean": 45.73275768638084, + "p50": 46.6100753353155, + "p90": 49.16386275490361, + "p95": 50.008765212349545, + "p99": 51.65800156214224 + }, + "itl_ms": { + "mean": 45.52147672991281, + "p50": 46.38860522058796, + "p90": 48.98359987373919, + "p95": 49.841784776414414, + "p99": 51.40792481203061 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_256.jsonl" + }, + { + "name": "c128_i2048_o4096", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 204.97591403801925, + "request_throughput": 12.48927227383978, + "input_token_throughput": 25578.02961682387, + "output_token_throughput": 2696.3216756164234, + "total_token_throughput": 28274.351292440293, + "total_input_tokens": 5242880, + "total_output_tokens": 552681, + "e2e_ms": { + "mean": 9834.139892057077, + "p50": 9220.157204486895, + "p90": 13741.088272898922, + "p95": 15705.61146168039, + "p99": 22444.27695973601 + }, + "ttft_ms": { + "mean": 369.64516925165753, + "p50": 241.21888101217337, + "p90": 378.9612676715477, + "p95": 497.4736295611367, + "p99": 4203.656943541137 + }, + "tpot_ms": { + "mean": 44.063288383150805, + "p50": 44.78743687857023, + "p90": 47.155220997591535, + "p95": 47.89839114115504, + "p99": 49.488713175876434 + }, + "itl_ms": { + "mean": 43.843357811756334, + "p50": 44.5406747903975, + "p90": 46.96878799674696, + "p95": 47.61324272796249, + "p99": 49.25185357024085 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_4096.jsonl" + }, + { + "name": "c128_i2048_o512", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 205.58478230598848, + "request_throughput": 12.45228353618968, + "input_token_throughput": 25502.276682116466, + "output_token_throughput": 2682.2413303882827, + "total_token_throughput": 28184.51801250475, + "total_input_tokens": 5242880, + "total_output_tokens": 551428, + "e2e_ms": { + "mean": 9849.413630793037, + "p50": 9248.099040007219, + "p90": 14181.897066975942, + "p95": 16273.27139499248, + "p99": 21537.212672340098 + }, + "ttft_ms": { + "mean": 364.9559352597862, + "p50": 238.98277248372324, + "p90": 354.852419818053, + "p95": 479.81080181489244, + "p99": 4198.306379870628 + }, + "tpot_ms": { + "mean": 44.31741461426594, + "p50": 45.06919488338426, + "p90": 47.75500610530777, + "p95": 48.38413202043307, + "p99": 49.462320926734186 + }, + "itl_ms": { + "mean": 44.087402562054, + "p50": 44.8524519679176, + "p90": 47.53244151442521, + "p95": 48.13061339723142, + "p99": 49.245863130187566 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_2048_512.jsonl" + }, + { + "name": "c128_i4096_o1024", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 299.5585728090373, + "request_throughput": 8.545907987190038, + "input_token_throughput": 35004.039115530395, + "output_token_throughput": 1823.4897932572892, + "total_token_throughput": 36827.52890878768, + "total_input_tokens": 10485760, + "total_output_tokens": 546242, + "e2e_ms": { + "mean": 14410.804073699523, + "p50": 13498.310438502813, + "p90": 20445.950364420423, + "p95": 23823.7266023556, + "p99": 34763.29548756821 + }, + "ttft_ms": { + "mean": 670.9425271239525, + "p50": 403.0677000118885, + "p90": 679.926768358564, + "p95": 898.5662415187111, + "p99": 8472.05125531822 + }, + "tpot_ms": { + "mean": 64.95321939396936, + "p50": 66.22150749587814, + "p90": 70.0543531767368, + "p95": 71.7235141039819, + "p99": 78.30801132371384 + }, + "itl_ms": { + "mean": 64.61322426057873, + "p50": 65.90216442502557, + "p90": 69.68458649417813, + "p95": 71.33039946453164, + "p99": 77.652479555507 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_1024.jsonl" + }, + { + "name": "c128_i4096_o128", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 252.68375409400323, + "request_throughput": 10.131240962359735, + "input_token_throughput": 41497.562981825475, + "output_token_throughput": 1283.4540992269378, + "total_token_throughput": 42781.017081052414, + "total_input_tokens": 10485760, + "total_output_tokens": 324308, + "e2e_ms": { + "mean": 12341.739772960294, + "p50": 12492.597976524848, + "p90": 13279.146579967346, + "p95": 13909.506630254331, + "p99": 19709.734952092404 + }, + "ttft_ms": { + "mean": 1110.3961577793825, + "p50": 913.746756996261, + "p90": 1339.3651077931281, + "p95": 1837.394617483369, + "p99": 8464.328227938386 + }, + "tpot_ms": { + "mean": 89.2549826316773, + "p50": 91.65863783074008, + "p90": 97.26721230404054, + "p95": 99.2759227111943, + "p99": 101.99371619496534 + }, + "itl_ms": { + "mean": 89.17840043206004, + "p50": 91.65084444093665, + "p90": 97.2365819717883, + "p95": 99.26035149961592, + "p99": 101.91815370646901 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_128.jsonl" + }, + { + "name": "c128_i4096_o2048", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 299.51152995903976, + "request_throughput": 8.54725025226941, + "input_token_throughput": 35009.537033295506, + "output_token_throughput": 1841.885686589241, + "total_token_throughput": 36851.42271988475, + "total_input_tokens": 10485760, + "total_output_tokens": 551666, + "e2e_ms": { + "mean": 14467.323276237676, + "p50": 13327.958792971913, + "p90": 20516.120994987432, + "p95": 24917.877360078277, + "p99": 37076.16252239852 + }, + "ttft_ms": { + "mean": 673.6397270240786, + "p50": 404.4142944912892, + "p90": 694.2803778627421, + "p95": 903.5128011921189, + "p99": 8456.423838282935 + }, + "tpot_ms": { + "mean": 64.31002423542024, + "p50": 65.36100197871913, + "p90": 69.56556259883966, + "p95": 71.20312924578917, + "p99": 77.97612688308777 + }, + "itl_ms": { + "mean": 63.97966287460745, + "p50": 65.00690745694432, + "p90": 69.20030281423097, + "p95": 70.85190648050559, + "p99": 77.09933678045897 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_2048.jsonl" + }, + { + "name": "c128_i4096_o256", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 288.9844872049871, + "request_throughput": 8.85860699569005, + "input_token_throughput": 36284.85425434644, + "output_token_throughput": 1743.9586632292514, + "total_token_throughput": 38028.81291757569, + "total_input_tokens": 10485760, + "total_output_tokens": 503977, + "e2e_ms": { + "mean": 14052.644563852664, + "p50": 14011.578131030547, + "p90": 18207.99634809373, + "p95": 18727.13703426707, + "p99": 21844.01485296432 + }, + "ttft_ms": { + "mean": 688.4898013677685, + "p50": 420.76750652631745, + "p90": 717.7646088181062, + "p95": 1029.3209117808167, + "p99": 8474.333298587006 + }, + "tpot_ms": { + "mean": 68.27357329740917, + "p50": 69.55606692148757, + "p90": 73.2924286689247, + "p95": 74.6707608597957, + "p99": 80.07070039799193 + }, + "itl_ms": { + "mean": 67.9460994704431, + "p50": 69.2711814984261, + "p90": 72.90093740416758, + "p95": 74.33009236125588, + "p99": 79.8225049918672 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_256.jsonl" + }, + { + "name": "c128_i4096_o4096", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 297.6913547680015, + "request_throughput": 8.599510731492602, + "input_token_throughput": 35223.5959561937, + "output_token_throughput": 1820.8355443255352, + "total_token_throughput": 37044.43150051923, + "total_input_tokens": 10485760, + "total_output_tokens": 542047, + "e2e_ms": { + "mean": 14368.531745259383, + "p50": 13428.582592488965, + "p90": 20025.193709513405, + "p95": 24055.67811630026, + "p99": 33864.98227334923 + }, + "ttft_ms": { + "mean": 679.8773801880316, + "p50": 405.75079250265844, + "p90": 709.7748359898104, + "p95": 979.6798837225666, + "p99": 8471.744555426993 + }, + "tpot_ms": { + "mean": 65.1796741939593, + "p50": 66.18706630407843, + "p90": 70.68885655515605, + "p95": 72.12222651762532, + "p99": 79.0404013413206 + }, + "itl_ms": { + "mean": 64.83921205698559, + "p50": 65.83542271383942, + "p90": 70.3737927563315, + "p95": 71.8124721641636, + "p99": 78.200747942912 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_4096.jsonl" + }, + { + "name": "c128_i4096_o512", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 296.8965953789884, + "request_throughput": 8.62253067177197, + "input_token_throughput": 35317.88563157799, + "output_token_throughput": 1815.7904414896925, + "total_token_throughput": 37133.67607306768, + "total_input_tokens": 10485760, + "total_output_tokens": 539102, + "e2e_ms": { + "mean": 14377.251431213948, + "p50": 13426.22295350884, + "p90": 20687.27437831694, + "p95": 24981.060059840107, + "p99": 33674.280850780175 + }, + "ttft_ms": { + "mean": 666.1897538902394, + "p50": 402.21049898536876, + "p90": 685.1608576835132, + "p95": 888.769919294283, + "p99": 8469.317924450152 + }, + "tpot_ms": { + "mean": 65.45340909588701, + "p50": 66.52154384660442, + "p90": 70.71509014012665, + "p95": 72.23137798844691, + "p99": 78.81060099442449 + }, + "itl_ms": { + "mean": 65.12317600401387, + "p50": 66.27417706121985, + "p90": 70.3212920331301, + "p95": 71.8430878627161, + "p99": 78.63558289697215 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_4096_512.jsonl" + }, + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 155.28525411698502, + "request_throughput": 16.485789423839368, + "input_token_throughput": 8440.724185005756, + "output_token_throughput": 3401.044117184036, + "total_token_throughput": 11841.768302189792, + "total_input_tokens": 1310720, + "total_output_tokens": 528132, + "e2e_ms": { + "mean": 7433.739403959908, + "p50": 7612.308174022473, + "p90": 9559.700762154534, + "p95": 9800.363442124217, + "p99": 10039.981560425947 + }, + "ttft_ms": { + "mean": 216.11621198128432, + "p50": 176.26684298738837, + "p90": 270.79010872403154, + "p95": 439.64523445174757, + "p99": 1351.9783651607574 + }, + "tpot_ms": { + "mean": 35.17330734203888, + "p50": 36.002418438770434, + "p90": 38.22760091307209, + "p95": 38.71466623999261, + "p99": 39.83368630610904 + }, + "itl_ms": { + "mean": 35.03133331348015, + "p50": 35.86981591799907, + "p90": 38.09707126092343, + "p95": 38.51183454514206, + "p99": 39.7125292160622 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_128_512_256.jsonl" + }, + { + "name": "c16_i512_o256", + "config": { + "concurrency": 16, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 60.82467504602391, + "request_throughput": 5.261022763506211, + "input_token_throughput": 2693.64365491518, + "output_token_throughput": 1089.672899195088, + "total_token_throughput": 3783.316554110268, + "total_input_tokens": 163840, + "total_output_tokens": 66279, + "e2e_ms": { + "mean": 2748.5973666503924, + "p50": 2736.172020493541, + "p90": 3588.4529045782983, + "p95": 4105.760691943578, + "p99": 4640.225247456692 + }, + "ttft_ms": { + "mean": 145.5557072244119, + "p50": 118.74175499542616, + "p90": 195.0831679045223, + "p95": 311.20013739855494, + "p99": 388.93986369890627 + }, + "tpot_ms": { + "mean": 12.620319833878325, + "p50": 12.123861945206848, + "p90": 15.690948749874153, + "p95": 16.997498412684262, + "p99": 17.834258368315975 + }, + "itl_ms": { + "mean": 12.568962422917354, + "p50": 12.075563610004078, + "p90": 15.655471615624732, + "p95": 16.959495267951805, + "p99": 17.8226512715222 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_16_512_256.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.340283242985606, + "request_throughput": 0.38211485992828104, + "input_token_throughput": 391.2856165665598, + "output_token_throughput": 85.13519079202102, + "total_token_throughput": 476.4208073585808, + "total_input_tokens": 20480, + "total_output_tokens": 4456, + "e2e_ms": { + "mean": 1666.6323251556605, + "p50": 1607.7605970203876, + "p90": 2214.885215985123, + "p95": 2301.1320795718348, + "p99": 2358.686553525622 + }, + "ttft_ms": { + "mean": 115.80964514578227, + "p50": 114.3495729775168, + "p90": 116.58464862266555, + "p95": 118.25840408855585, + "p99": 140.49979043367782 + }, + "tpot_ms": { + "mean": 6.992964128556314, + "p50": 6.993668697891669, + "p90": 6.99977319934427, + "p95": 7.0010958709895394, + "p99": 7.001298266553494 + }, + "itl_ms": { + "mean": 6.958230151605854, + "p50": 6.958213456912434, + "p90": 6.961470347430554, + "p95": 6.962435742448271, + "p99": 6.963362603158629 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o128", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 32.71276432002196, + "request_throughput": 0.611382144423635, + "input_token_throughput": 626.0553158898023, + "output_token_throughput": 78.07349984289819, + "total_token_throughput": 704.1288157327004, + "total_input_tokens": 20480, + "total_output_tokens": 2554, + "e2e_ms": { + "mean": 997.0761572010815, + "p50": 998.1900005077478, + "p90": 1002.62519911048, + "p95": 1004.8363747657277, + "p99": 1011.9732421101071 + }, + "ttft_ms": { + "mean": 115.304818644654, + "p50": 115.1561715232674, + "p90": 118.39750949875452, + "p95": 122.94176129507832, + "p99": 128.4698994399514 + }, + "tpot_ms": { + "mean": 6.95965548413073, + "p50": 6.958289842596131, + "p90": 6.960662914757199, + "p95": 6.970090178271585, + "p99": 7.007105844121334 + }, + "itl_ms": { + "mean": 6.956049581701196, + "p50": 6.9574823148830935, + "p90": 6.959047572280273, + "p95": 6.959772129051799, + "p99": 6.965634709180105 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_128.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.49656022601994, + "request_throughput": 0.41240038276507224, + "input_token_throughput": 422.29799195143397, + "output_token_throughput": 83.01619705060904, + "total_token_throughput": 505.314189002043, + "total_input_tokens": 20480, + "total_output_tokens": 4026, + "e2e_ms": { + "mean": 1516.367995305336, + "p50": 1535.2776270301547, + "p90": 1693.6148332140874, + "p95": 1700.7698216184508, + "p99": 1778.3540971053296 + }, + "ttft_ms": { + "mean": 115.62368175073061, + "p50": 115.08111699367873, + "p90": 118.04690642748028, + "p95": 119.49695778021123, + "p99": 126.28927154641131 + }, + "tpot_ms": { + "mean": 6.993729136018383, + "p50": 6.993365077923091, + "p90": 6.998930970311811, + "p95": 7.004203024148848, + "p99": 7.006087367499652 + }, + "itl_ms": { + "mean": 6.955256937332673, + "p50": 6.955609315216483, + "p90": 6.9578465340344176, + "p95": 6.958589018775167, + "p99": 6.959668077607412 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.48484487499809, + "request_throughput": 0.4125000307119326, + "input_token_throughput": 422.40003144901897, + "output_token_throughput": 81.26250605025072, + "total_token_throughput": 503.6625374992697, + "total_input_tokens": 20480, + "total_output_tokens": 3940, + "e2e_ms": { + "mean": 1485.3911130456254, + "p50": 1523.8948875048663, + "p90": 1818.4952258015983, + "p95": 1837.5514468731126, + "p99": 1880.363019790384 + }, + "ttft_ms": { + "mean": 114.81126984290313, + "p50": 113.8783980277367, + "p90": 117.8437611728441, + "p95": 120.57862681685947, + "p99": 125.23157255607656 + }, + "tpot_ms": { + "mean": 6.994496555731443, + "p50": 6.995160303404432, + "p90": 7.0082143222680315, + "p95": 7.010770282737416, + "p99": 7.011674420422894 + }, + "itl_ms": { + "mean": 6.955705517495981, + "p50": 6.954842998457913, + "p90": 6.95964445904527, + "p95": 6.959962120696949, + "p99": 6.963102689140168 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_256.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 51.98334731196519, + "request_throughput": 0.3847385948422088, + "input_token_throughput": 393.9723211184218, + "output_token_throughput": 82.7187978910749, + "total_token_throughput": 476.69111900949673, + "total_input_tokens": 20480, + "total_output_tokens": 4300, + "e2e_ms": { + "mean": 1612.523135956144, + "p50": 1597.956053999951, + "p90": 2029.4950069335755, + "p95": 2093.7656364869326, + "p99": 2163.9699145313352 + }, + "ttft_ms": { + "mean": 116.50667509820778, + "p50": 114.9420035071671, + "p90": 118.94925460801461, + "p95": 126.80927838664503, + "p99": 137.82952686306086 + }, + "tpot_ms": { + "mean": 6.991852548113963, + "p50": 6.98992295172064, + "p90": 7.002194919252066, + "p95": 7.004974589785324, + "p99": 7.007806217299085 + }, + "itl_ms": { + "mean": 6.954919126074468, + "p50": 6.955506687569439, + "p90": 6.958137890710187, + "p95": 6.959299433278432, + "p99": 6.959750128390998 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_4096.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 49.68123743298929, + "request_throughput": 0.4025664623788058, + "input_token_throughput": 412.2280574758971, + "output_token_throughput": 78.90302662624593, + "total_token_throughput": 491.131084102143, + "total_input_tokens": 20480, + "total_output_tokens": 3920, + "e2e_ms": { + "mean": 1497.0933472388424, + "p50": 1472.0589299686253, + "p90": 1865.7209743745627, + "p95": 2052.554446668364, + "p99": 2150.5699093313888 + }, + "ttft_ms": { + "mean": 132.68365324183833, + "p50": 115.42460750206374, + "p90": 173.59126399969682, + "p95": 174.47906527668238, + "p99": 175.38653943687677 + }, + "tpot_ms": { + "mean": 6.998256694810179, + "p50": 6.995677604377155, + "p90": 7.012659735752915, + "p95": 7.016148772127566, + "p99": 7.016167255125867 + }, + "itl_ms": { + "mean": 6.957685190034067, + "p50": 6.957637038476368, + "p90": 6.964746559207174, + "p95": 6.965498266336599, + "p99": 6.965703383228294 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_1024_512.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 257.4472376179765, + "request_throughput": 0.07768582092800626, + "input_token_throughput": 10182.435920675636, + "output_token_throughput": 8.902795078349518, + "total_token_throughput": 10191.338715753987, + "total_input_tokens": 2621440, + "total_output_tokens": 2292, + "e2e_ms": { + "mean": 7290.28443134448, + "p50": 7286.690048524179, + "p90": 7319.372808613116, + "p95": 7408.333866103203, + "p99": 7425.827623613295 + }, + "ttft_ms": { + "mean": 6471.478171434137, + "p50": 6459.610841469839, + "p90": 6570.740136282984, + "p95": 6574.879066209542, + "p99": 6590.681217216188 + }, + "tpot_ms": { + "mean": 7.209416115326391, + "p50": 7.197772930998452, + "p90": 7.2473051953872965, + "p95": 7.251998598967419, + "p99": 7.316937038715112 + }, + "itl_ms": { + "mean": 7.187574219107276, + "p50": 7.195279056382976, + "p90": 7.206139789597164, + "p95": 7.207912252563582, + "p99": 7.21993116777318 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 259.43278859899146, + "request_throughput": 0.07709125784757397, + "input_token_throughput": 10104.505348597215, + "output_token_throughput": 13.109368396979955, + "total_token_throughput": 10117.614716994196, + "total_input_tokens": 2621440, + "total_output_tokens": 3401, + "e2e_ms": { + "mean": 7749.514106288552, + "p50": 7651.893796457443, + "p90": 8206.727583683096, + "p95": 8208.241063222522, + "p99": 8219.552055849927 + }, + "ttft_ms": { + "mean": 6523.804933592328, + "p50": 6512.4543030106, + "p90": 6598.4229369903915, + "p95": 6605.214903116575, + "p99": 6683.361206200789 + }, + "tpot_ms": { + "mean": 7.253172623286035, + "p50": 7.254124181907761, + "p90": 7.272623823075956, + "p95": 7.2848265816878275, + "p99": 7.296334035252958 + }, + "itl_ms": { + "mean": 7.179794252086232, + "p50": 7.175094242766207, + "p90": 7.214930474917503, + "p95": 7.216970495949786, + "p99": 7.221291095815641 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 281.0289259409765, + "request_throughput": 0.07116705133833992, + "input_token_throughput": 9328.00775301889, + "output_token_throughput": 15.400549909616759, + "total_token_throughput": 9343.408302928507, + "total_input_tokens": 2621440, + "total_output_tokens": 4328, + "e2e_ms": { + "mean": 8083.131315253559, + "p50": 7868.732059490867, + "p90": 9030.448771850208, + "p95": 9930.97106947098, + "p99": 10147.713469895534 + }, + "ttft_ms": { + "mean": 6520.167630907963, + "p50": 6511.3907440099865, + "p90": 6591.417226468911, + "p95": 6595.290473787463, + "p99": 6639.392100346158 + }, + "tpot_ms": { + "mean": 7.255575324965767, + "p50": 7.2530971403025575, + "p90": 7.261777192937413, + "p95": 7.272023605883151, + "p99": 7.29306066564882 + }, + "itl_ms": { + "mean": 7.187150700754481, + "p50": 7.189161557386939, + "p90": 7.219701732356514, + "p95": 7.228312448436723, + "p99": 7.247371476856312 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_131072_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 85.55190685600974, + "request_throughput": 0.2337762036521465, + "input_token_throughput": 3830.189320636768, + "output_token_throughput": 66.12359920300963, + "total_token_throughput": 3896.312919839778, + "total_input_tokens": 327680, + "total_output_tokens": 5657, + "e2e_ms": { + "mean": 2761.828099846025, + "p50": 2430.3790545091033, + "p90": 3463.2150286342962, + "p95": 4275.721653646907, + "p99": 5529.526115544721 + }, + "ttft_ms": { + "mean": 761.5903040976264, + "p50": 760.6050779868383, + "p90": 766.9433312665205, + "p95": 767.1664872235851, + "p99": 767.5652326649288 + }, + "tpot_ms": { + "mean": 7.09461820686801, + "p50": 7.094541371586382, + "p90": 7.10644475822362, + "p95": 7.10756015048735, + "p99": 7.107590855281464 + }, + "itl_ms": { + "mean": 7.060456193738931, + "p50": 7.055744335329496, + "p90": 7.081214189454361, + "p95": 7.090980342578718, + "p99": 7.095196453397732 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 54.062904359947424, + "request_throughput": 0.36993942957339576, + "input_token_throughput": 6061.087614130516, + "output_token_throughput": 46.98230755582126, + "total_token_throughput": 6108.069921686337, + "total_input_tokens": 327680, + "total_output_tokens": 2540, + "e2e_ms": { + "mean": 1645.3096443059621, + "p50": 1645.1492685009725, + "p90": 1648.5932470939588, + "p95": 1649.6672159439186, + "p99": 1651.5440808032872 + }, + "ttft_ms": { + "mean": 758.9813886006596, + "p50": 759.2984469956718, + "p90": 761.3394041894935, + "p95": 762.0397499384126, + "p99": 764.5994139945833 + }, + "tpot_ms": { + "mean": 7.03435123575637, + "p50": 7.034438234219875, + "p90": 7.0417456302998795, + "p95": 7.043358887289835, + "p99": 7.044686933047145 + }, + "itl_ms": { + "mean": 7.0338002467502125, + "p50": 7.033853519787376, + "p90": 7.041230728519001, + "p95": 7.042800428369822, + "p99": 7.043882879430414 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_128.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 90.19640143198194, + "request_throughput": 0.22173833636902035, + "input_token_throughput": 3632.9609030700294, + "output_token_throughput": 62.70760152515896, + "total_token_throughput": 3695.6685045951886, + "total_input_tokens": 327680, + "total_output_tokens": 5656, + "e2e_ms": { + "mean": 2762.5432386033935, + "p50": 2452.210796473082, + "p90": 3715.490449598292, + "p95": 3858.2562830328243, + "p99": 4063.0771798180645 + }, + "ttft_ms": { + "mean": 762.252552350401, + "p50": 762.4800890043844, + "p90": 767.4323470098898, + "p95": 769.1343927930575, + "p99": 769.4857225881424 + }, + "tpot_ms": { + "mean": 7.097202645035762, + "p50": 7.097632476244429, + "p90": 7.107065638170411, + "p95": 7.107487881600403, + "p99": 7.107727561069631 + }, + "itl_ms": { + "mean": 7.064960211576161, + "p50": 7.06291316644792, + "p90": 7.086573517755428, + "p95": 7.088723471803392, + "p99": 7.09011580964756 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_2048.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 73.12888823298272, + "request_throughput": 0.2734897313942695, + "input_token_throughput": 4480.855759163712, + "output_token_throughput": 56.18846531495267, + "total_token_throughput": 4537.044224478665, + "total_input_tokens": 327680, + "total_output_tokens": 4109, + "e2e_ms": { + "mean": 2205.644827496144, + "p50": 2216.0204490064643, + "p90": 2557.4225594173186, + "p95": 2558.677955370513, + "p99": 2568.8925430789823 + }, + "ttft_ms": { + "mean": 757.2442296514055, + "p50": 759.4909319886938, + "p90": 765.4850837308913, + "p95": 769.1775480052456, + "p99": 772.9351856047288 + }, + "tpot_ms": { + "mean": 7.086063474876053, + "p50": 7.086371160285189, + "p90": 7.106006101920233, + "p95": 7.107053158383294, + "p99": 7.108264589840503 + }, + "itl_ms": { + "mean": 7.0536245266423645, + "p50": 7.0581677382566905, + "p90": 7.074847337643169, + "p95": 7.076649383083078, + "p99": 7.076766213604275 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 75.42876927595353, + "request_throughput": 0.2651508196671047, + "input_token_throughput": 4344.231029425843, + "output_token_throughput": 53.958191802255804, + "total_token_throughput": 4398.189221228099, + "total_input_tokens": 327680, + "total_output_tokens": 4070, + "e2e_ms": { + "mean": 2198.956436300068, + "p50": 2213.818849500967, + "p90": 2626.4926128555094, + "p95": 2832.268335088156, + "p99": 2889.4597958168015 + }, + "ttft_ms": { + "mean": 761.9774985505501, + "p50": 761.5274374838918, + "p90": 764.6430697699543, + "p95": 768.2955486234277, + "p99": 778.232998540625 + }, + "tpot_ms": { + "mean": 7.0961170639464255, + "p50": 7.095928777097879, + "p90": 7.106023095899129, + "p95": 7.106170595785851, + "p99": 7.107090797877847 + }, + "itl_ms": { + "mean": 7.053604431683894, + "p50": 7.054318090364773, + "p90": 7.067077616323665, + "p95": 7.068994884135261, + "p99": 7.0774931702040575 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_16384_512.jsonl" + }, + { + "name": "c1_i2048_o1024", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.1534125340404, + "request_throughput": 0.3834840143383916, + "input_token_throughput": 785.375261365026, + "output_token_throughput": 81.93135966339736, + "total_token_throughput": 867.3066210284234, + "total_input_tokens": 40960, + "total_output_tokens": 4273, + "e2e_ms": { + "mean": 1639.0007649490144, + "p50": 1520.7974360091612, + "p90": 2190.022072021384, + "p95": 2263.6719663161794, + "p99": 3129.8387580644326 + }, + "ttft_ms": { + "mean": 135.1462520484347, + "p50": 133.8202319748234, + "p90": 138.35612262482755, + "p95": 140.2339472057065, + "p99": 150.81769502896347 + }, + "tpot_ms": { + "mean": 7.0747086929852685, + "p50": 7.0748485663652305, + "p90": 7.088887935875075, + "p95": 7.091126444031059, + "p99": 7.099713838744482 + }, + "itl_ms": { + "mean": 7.03472727239529, + "p50": 7.033652576394227, + "p90": 7.045419228656695, + "p95": 7.045716443511645, + "p99": 7.0469850743595 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_1024.jsonl" + }, + { + "name": "c1_i2048_o128", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 33.24076693301322, + "request_throughput": 0.6016708351014882, + "input_token_throughput": 1232.2218702878479, + "output_token_throughput": 74.96818605364544, + "total_token_throughput": 1307.1900563414933, + "total_input_tokens": 40960, + "total_output_tokens": 2492, + "e2e_ms": { + "mean": 1006.6183711052872, + "p50": 1025.5568735010456, + "p90": 1031.097009422956, + "p95": 1053.4016749501461, + "p99": 1075.8364333986538 + }, + "ttft_ms": { + "mean": 136.94067875039764, + "p50": 133.35946449660696, + "p90": 137.41420948063026, + "p95": 159.17179067619148, + "p99": 181.960963755846 + }, + "tpot_ms": { + "mean": 7.038047122125526, + "p50": 7.034313641800216, + "p90": 7.047546141091331, + "p95": 7.10328894771225, + "p99": 7.104833651601928 + }, + "itl_ms": { + "mean": 7.029471314430009, + "p50": 7.0264248858484635, + "p90": 7.037256039242722, + "p95": 7.0381679429625, + "p99": 7.040319012296496 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_128.jsonl" + }, + { + "name": "c1_i2048_o2048", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 51.476043572009075, + "request_throughput": 0.38853024848388545, + "input_token_throughput": 795.7099488949974, + "output_token_throughput": 84.60246160736605, + "total_token_throughput": 880.3124105023635, + "total_input_tokens": 40960, + "total_output_tokens": 4355, + "e2e_ms": { + "mean": 1667.398910401971, + "p50": 1539.4108625187073, + "p90": 2376.00737574394, + "p95": 3179.9570861068787, + "p99": 3366.919990839087 + }, + "ttft_ms": { + "mean": 134.55304539820645, + "p50": 133.54374349000864, + "p90": 135.26073460816406, + "p95": 137.97220440756064, + "p99": 148.68526168575042 + }, + "tpot_ms": { + "mean": 7.0742273781296205, + "p50": 7.074927825367073, + "p90": 7.082008953453719, + "p95": 7.089882750816399, + "p99": 7.095734023629707 + }, + "itl_ms": { + "mean": 7.034275217543682, + "p50": 7.034182887314636, + "p90": 7.044180244236356, + "p95": 7.0465115520927295, + "p99": 7.051333929426694 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_2048.jsonl" + }, + { + "name": "c1_i2048_o256", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 48.447020565974526, + "request_throughput": 0.4128220841313504, + "input_token_throughput": 845.4596283010056, + "output_token_throughput": 79.01414690274046, + "total_token_throughput": 924.4737752037462, + "total_input_tokens": 40960, + "total_output_tokens": 3828, + "e2e_ms": { + "mean": 1479.6458801545668, + "p50": 1452.7736474992707, + "p90": 1930.3096152900252, + "p95": 1930.9429561020806, + "p99": 1931.7471896251664 + }, + "ttft_ms": { + "mean": 133.364660301595, + "p50": 133.06230350281112, + "p90": 134.4277158670593, + "p95": 134.8362378514139, + "p99": 138.7125235464191 + }, + "tpot_ms": { + "mean": 7.072852898068682, + "p50": 7.075266202282437, + "p90": 7.0822706008898235, + "p95": 7.0830494113516, + "p99": 7.091806624947279 + }, + "itl_ms": { + "mean": 7.0355477412239775, + "p50": 7.036148776481396, + "p90": 7.045096764934486, + "p95": 7.0460076927887645, + "p99": 7.047783690598319 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_256.jsonl" + }, + { + "name": "c1_i2048_o4096", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 52.65344714699313, + "request_throughput": 0.37984217717343005, + "input_token_throughput": 777.9167788511847, + "output_token_throughput": 77.54478046995574, + "total_token_throughput": 855.4615593211405, + "total_input_tokens": 40960, + "total_output_tokens": 4083, + "e2e_ms": { + "mean": 1571.4832365047187, + "p50": 1578.1495150004048, + "p90": 1986.1723132315092, + "p95": 2033.6844808247406, + "p99": 2095.4177425953094 + }, + "ttft_ms": { + "mean": 134.26353290269617, + "p50": 133.42173700220883, + "p90": 134.74032204248942, + "p95": 137.78332775691524, + "p99": 149.1545207169838 + }, + "tpot_ms": { + "mean": 7.075507301509984, + "p50": 7.0742508989005675, + "p90": 7.083799128494104, + "p95": 7.085761582440888, + "p99": 7.097104248244071 + }, + "itl_ms": { + "mean": 7.036068429931236, + "p50": 7.036433208274854, + "p90": 7.044877956843721, + "p95": 7.045143621820257, + "p99": 7.0458232713827575 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_4096.jsonl" + }, + { + "name": "c1_i2048_o512", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.388948728039395, + "request_throughput": 0.3969124283172595, + "input_token_throughput": 812.8766531937474, + "output_token_throughput": 79.10464696362982, + "total_token_throughput": 891.9813001573773, + "total_input_tokens": 40960, + "total_output_tokens": 3986, + "e2e_ms": { + "mean": 1537.0369591546478, + "p50": 1504.8605250194669, + "p90": 2335.837259324035, + "p95": 2769.402894700761, + "p99": 2894.6374381514033 + }, + "ttft_ms": { + "mean": 133.85857905086596, + "p50": 133.14539249404334, + "p90": 134.4122292357497, + "p95": 135.40067606954835, + "p99": 147.36564562539567 + }, + "tpot_ms": { + "mean": 7.080310007154715, + "p50": 7.080751560016797, + "p90": 7.098766246633662, + "p95": 7.102012714910587, + "p99": 7.103908377730314 + }, + "itl_ms": { + "mean": 7.035449573803811, + "p50": 7.035650407287871, + "p90": 7.044265468773851, + "p95": 7.046153439982385, + "p99": 7.049547889021412 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_2048_512.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 556.8975440699724, + "request_throughput": 0.03591324869891519, + "input_token_throughput": 9414.442666928424, + "output_token_throughput": 6.2722488852655385, + "total_token_throughput": 9420.714915813689, + "total_input_tokens": 5242880, + "total_output_tokens": 3493, + "e2e_ms": { + "mean": 16530.717764247674, + "p50": 16664.499283506302, + "p90": 17023.02305897465, + "p95": 17025.054730603006, + "p99": 17032.912425303366 + }, + "ttft_ms": { + "mean": 15234.609863755759, + "p50": 15248.796680971282, + "p90": 15315.202502999455, + "p95": 15316.530187413446, + "p99": 15320.265578298713 + }, + "tpot_ms": { + "mean": 7.4726540033093825, + "p50": 7.4731072332071005, + "p90": 7.507545618103787, + "p95": 7.547523731020181, + "p99": 7.567509443686337 + }, + "itl_ms": { + "mean": 7.3375862011256014, + "p50": 7.328241241606573, + "p90": 7.390136176649624, + "p95": 7.407534864068883, + "p99": 7.539143762656007 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_262144_256.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.77059198997449, + "request_throughput": 0.3939288319495927, + "input_token_throughput": 1613.5324956655318, + "output_token_throughput": 78.27365890838408, + "total_token_throughput": 1691.8061545739158, + "total_input_tokens": 81920, + "total_output_tokens": 3974, + "e2e_ms": { + "mean": 1622.7367079496616, + "p50": 1548.5019845073111, + "p90": 1970.093786006328, + "p95": 2090.628466138151, + "p99": 2641.6305804229332 + }, + "ttft_ms": { + "mean": 223.9771443477366, + "p50": 223.59500601305626, + "p90": 227.59674831759185, + "p95": 228.83622981898952, + "p99": 229.9823219521204 + }, + "tpot_ms": { + "mean": 7.076595673752794, + "p50": 7.081092141277011, + "p90": 7.086373823695292, + "p95": 7.087853686347765, + "p99": 7.090011781365773 + }, + "itl_ms": { + "mean": 7.034033030946358, + "p50": 7.037121284554964, + "p90": 7.044552039641219, + "p95": 7.0486115542576515, + "p99": 7.053144998174831 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 36.183199760038406, + "request_throughput": 0.5527427129893714, + "input_token_throughput": 2264.034152404465, + "output_token_throughput": 70.1706874140007, + "total_token_throughput": 2334.204839818466, + "total_input_tokens": 81920, + "total_output_tokens": 2539, + "e2e_ms": { + "mean": 1108.6190134024946, + "p50": 1115.3957745118532, + "p90": 1117.3203395039309, + "p95": 1117.8947942622472, + "p99": 1119.9307620862965 + }, + "ttft_ms": { + "mean": 223.36841770156752, + "p50": 223.17008301615715, + "p90": 224.61351838428527, + "p95": 225.7140525529394, + "p99": 227.20984732557554 + }, + "tpot_ms": { + "mean": 7.029005154512127, + "p50": 7.0305118150734085, + "p90": 7.034080769586208, + "p95": 7.03638333245587, + "p99": 7.070559654767858 + }, + "itl_ms": { + "mean": 7.02481579687929, + "p50": 7.025486673228443, + "p90": 7.033457389691713, + "p95": 7.033528619208447, + "p99": 7.033584231027681 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_128.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 54.08545239997329, + "request_throughput": 0.3697852030911343, + "input_token_throughput": 1514.640191861286, + "output_token_throughput": 78.3389952748568, + "total_token_throughput": 1592.9791871361429, + "total_input_tokens": 81920, + "total_output_tokens": 4237, + "e2e_ms": { + "mean": 1715.1450212550117, + "p50": 1574.5812459790614, + "p90": 2619.9436117953155, + "p95": 2736.4090675429907, + "p99": 2965.236396695254 + }, + "ttft_ms": { + "mean": 223.30448884749785, + "p50": 223.00233450368978, + "p90": 224.80262753088027, + "p95": 226.85476079350337, + "p99": 227.96985933324322 + }, + "tpot_ms": { + "mean": 7.075842410128919, + "p50": 7.075092399596053, + "p90": 7.085437742444061, + "p95": 7.0869074893536945, + "p99": 7.087335458129968 + }, + "itl_ms": { + "mean": 7.034790197294845, + "p50": 7.036254612914846, + "p90": 7.052687796748768, + "p95": 7.054119502062382, + "p99": 7.054655302466439 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 53.93014029797632, + "request_throughput": 0.37085013852171417, + "input_token_throughput": 1519.0021673849412, + "output_token_throughput": 77.17391382636872, + "total_token_throughput": 1596.17608121131, + "total_input_tokens": 81920, + "total_output_tokens": 4162, + "e2e_ms": { + "mean": 1687.853766803164, + "p50": 1731.4121435047127, + "p90": 2022.2392167255748, + "p95": 2022.9304053325905, + "p99": 2023.168032289832 + }, + "ttft_ms": { + "mean": 224.11002360167913, + "p50": 223.77661100472324, + "p90": 225.24597371229902, + "p95": 227.58357299608178, + "p99": 229.3452226149384 + }, + "tpot_ms": { + "mean": 7.069597663492597, + "p50": 7.071157336921616, + "p90": 7.084982468067765, + "p95": 7.0854949991775005, + "p99": 7.086657547444251 + }, + "itl_ms": { + "mean": 7.036856933328044, + "p50": 7.036261300358248, + "p90": 7.0516792783632365, + "p95": 7.053535119733116, + "p99": 7.053621047452621 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_256.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 50.60050758899888, + "request_throughput": 0.39525295205434313, + "input_token_throughput": 1618.9560916145895, + "output_token_throughput": 72.98345759683446, + "total_token_throughput": 1691.9395492114238, + "total_input_tokens": 81920, + "total_output_tokens": 3693, + "e2e_ms": { + "mean": 1522.8591587103438, + "p50": 1464.6899780200329, + "p90": 1853.4205963020217, + "p95": 2146.742542926222, + "p99": 2282.4891966208816 + }, + "ttft_ms": { + "mean": 223.4261866658926, + "p50": 223.1349789944943, + "p90": 227.13108609896153, + "p95": 228.4300401690416, + "p99": 229.7299312404357 + }, + "tpot_ms": { + "mean": 7.07648349021855, + "p50": 7.077039959071561, + "p90": 7.086898134567167, + "p95": 7.090804919410713, + "p99": 7.091896303988687 + }, + "itl_ms": { + "mean": 7.031315753257085, + "p50": 7.035521328040964, + "p90": 7.0425933163626375, + "p95": 7.043398734457423, + "p99": 7.044492975297885 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_4096.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 59.7474733509589, + "request_throughput": 0.3347421887201698, + "input_token_throughput": 1371.1040049978155, + "output_token_throughput": 81.14150654576916, + "total_token_throughput": 1452.2455115435846, + "total_input_tokens": 81920, + "total_output_tokens": 4848, + "e2e_ms": { + "mean": 1931.5284742042422, + "p50": 1740.4890735051595, + "p90": 2566.8827370624062, + "p95": 2977.276942195022, + "p99": 3648.358510847319 + }, + "ttft_ms": { + "mean": 224.35228339745663, + "p50": 223.72989598079585, + "p90": 228.29745516064577, + "p95": 229.06503619451541, + "p99": 230.83653841691557 + }, + "tpot_ms": { + "mean": 7.072256846460429, + "p50": 7.074458655208373, + "p90": 7.080548995162574, + "p95": 7.081527911139119, + "p99": 7.081865160556928 + }, + "itl_ms": { + "mean": 7.03638093066817, + "p50": 7.042975338492516, + "p90": 7.048521073698659, + "p95": 7.052580513075507, + "p99": 7.055588117189501 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_4096_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 53.26067367795622, + "request_throughput": 0.3755115851694098, + "input_token_throughput": 192.26193160673782, + "output_token_throughput": 80.81009312845698, + "total_token_throughput": 273.0720247351948, + "total_input_tokens": 10240, + "total_output_tokens": 4304, + "e2e_ms": { + "mean": 1663.9683443499962, + "p50": 1712.9185860103462, + "p90": 1949.9001824238803, + "p95": 1951.3943302445114, + "p99": 1953.2585076522082 + }, + "ttft_ms": { + "mean": 167.28168494882993, + "p50": 171.281192015158, + "p90": 185.20025337347764, + "p95": 203.13406442292035, + "p99": 205.33247768878937 + }, + "tpot_ms": { + "mean": 6.990167993750641, + "p50": 6.991353931776988, + "p90": 7.007845179362607, + "p95": 7.016671783989656, + "p99": 7.025201123019247 + }, + "itl_ms": { + "mean": 6.959514433374882, + "p50": 6.960831024201191, + "p90": 6.963961682596841, + "p95": 6.965071484525725, + "p99": 6.965092276522976 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_512_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 1513.7487565820338, + "request_throughput": 0.013212232157441346, + "input_token_throughput": 6927.014773360608, + "output_token_throughput": 1.2042949611507787, + "total_token_throughput": 6928.21906832176, + "total_input_tokens": 10485760, + "total_output_tokens": 1823, + "e2e_ms": { + "mean": 41303.5668047989, + "p50": 41123.62798998947, + "p90": 41813.11989680398, + "p95": 41857.1446898015, + "p99": 41958.446165995556 + }, + "ttft_ms": { + "mean": 40602.843232947635, + "p50": 40378.582418488804, + "p90": 41078.4518414177, + "p95": 41121.70196585939, + "p99": 41219.488381977426 + }, + "tpot_ms": { + "mean": 7.778498289940974, + "p50": 7.751132273666659, + "p90": 7.8827576197313025, + "p95": 7.922383648177905, + "p99": 7.92777625971768 + }, + "itl_ms": { + "mean": 7.716799590558232, + "p50": 7.735913531474868, + "p90": 7.867651437648979, + "p95": 7.879696678362727, + "p99": 7.912987946224729 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_524288_128.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 152.80665369099006, + "request_throughput": 0.13088435298403017, + "input_token_throughput": 8577.636957161401, + "output_token_throughput": 26.28157807919326, + "total_token_throughput": 8603.918535240595, + "total_input_tokens": 1310720, + "total_output_tokens": 4016, + "e2e_ms": { + "mean": 4418.26631075819, + "p50": 4360.3685425186995, + "p90": 4756.268920592266, + "p95": 5282.474650835502, + "p99": 5593.151110982871 + }, + "ttft_ms": { + "mean": 2984.4894192006905, + "p50": 2984.1709710017312, + "p90": 2992.8966829786077, + "p95": 2994.563281320734, + "p99": 3006.297108255094 + }, + "tpot_ms": { + "mean": 7.17527447116709, + "p50": 7.174407352670137, + "p90": 7.1827685568605455, + "p95": 7.184976731507607, + "p99": 7.186551507442626 + }, + "itl_ms": { + "mean": 7.12250114574196, + "p50": 7.123122260997471, + "p90": 7.143086916873816, + "p95": 7.151460329389299, + "p99": 7.155490767844952 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_1024.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 134.86410969495773, + "request_throughput": 0.14829742357130435, + "input_token_throughput": 9718.819951169002, + "output_token_throughput": 17.89208415387787, + "total_token_throughput": 9736.71203532288, + "total_input_tokens": 1310720, + "total_output_tokens": 2413, + "e2e_ms": { + "mean": 3847.8309919009916, + "p50": 3835.5290399922524, + "p90": 3901.8099384149536, + "p95": 3911.314021665021, + "p99": 3932.2561667038826 + }, + "ttft_ms": { + "mean": 2996.6540380526567, + "p50": 2983.167622500332, + "p90": 3048.6084129835945, + "p95": 3058.3009802619927, + "p99": 3078.256532016676 + }, + "tpot_ms": { + "mean": 7.114025330224445, + "p50": 7.111448429107744, + "p90": 7.122448258696979, + "p95": 7.1562583283718695, + "p99": 7.1600122270900055 + }, + "itl_ms": { + "mean": 7.109220088670858, + "p50": 7.109864420878391, + "p90": 7.118059088340184, + "p95": 7.120027479602981, + "p99": 7.148371996643857 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 153.55778337997617, + "request_throughput": 0.13024413064436033, + "input_token_throughput": 8535.679345908798, + "output_token_throughput": 25.964167443953233, + "total_token_throughput": 8561.643513352752, + "total_input_tokens": 1310720, + "total_output_tokens": 3987, + "e2e_ms": { + "mean": 4407.236540847225, + "p50": 4463.37615000084, + "p90": 4759.197858988773, + "p95": 4762.50999016338, + "p99": 4778.89939242159 + }, + "ttft_ms": { + "mean": 2986.1355356988497, + "p50": 2985.1215340022463, + "p90": 2992.1448093722574, + "p95": 2994.618826545775, + "p99": 3008.904767688364 + }, + "tpot_ms": { + "mean": 7.16319049732031, + "p50": 7.168127065689811, + "p90": 7.181539436645955, + "p95": 7.1846426181983665, + "p99": 7.210727464676715 + }, + "itl_ms": { + "mean": 7.109022871854964, + "p50": 7.132144706104665, + "p90": 7.1512719032104926, + "p95": 7.154648859390686, + "p99": 7.159991177650125 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 157.15850195498206, + "request_throughput": 0.12726005752923877, + "input_token_throughput": 8340.115130236192, + "output_token_throughput": 27.70451452411528, + "total_token_throughput": 8367.819644760308, + "total_input_tokens": 1310720, + "total_output_tokens": 4354, + "e2e_ms": { + "mean": 4541.314341998077, + "p50": 4300.338625005679, + "p90": 5846.8827306700405, + "p95": 6107.6582527923165, + "p99": 6492.713085743016 + }, + "ttft_ms": { + "mean": 2986.0649044945603, + "p50": 2985.287106508622, + "p90": 2993.119363480946, + "p95": 2996.3409428397426, + "p99": 3023.154667772469 + }, + "tpot_ms": { + "mean": 7.175812514737212, + "p50": 7.174307613181792, + "p90": 7.18507248592225, + "p95": 7.18625519569333, + "p99": 7.200756394222195 + }, + "itl_ms": { + "mean": 7.1223437938975085, + "p50": 7.12593037796584, + "p90": 7.158473757668606, + "p95": 7.163340571236723, + "p99": 7.170335645946346 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_1_65536_512.jsonl" + }, + { + "name": "c2_i512_o256", + "config": { + "concurrency": 2, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 44.93843933503376, + "request_throughput": 0.8901065678268052, + "input_token_throughput": 455.73456272732426, + "output_token_throughput": 188.3242970879563, + "total_token_throughput": 644.0588598152806, + "total_input_tokens": 20480, + "total_output_tokens": 8463, + "e2e_ms": { + "mean": 1611.5773893747246, + "p50": 1769.6556154696736, + "p90": 1976.518541405676, + "p95": 2020.206581390812, + "p99": 2050.242396299145 + }, + "ttft_ms": { + "mean": 123.41228122531902, + "p50": 109.16157849715091, + "p90": 174.43694929243065, + "p95": 177.91075359564272, + "p99": 187.25801390013658 + }, + "tpot_ms": { + "mean": 7.0588251237143655, + "p50": 6.9910976638514235, + "p90": 7.33967459413643, + "p95": 7.48310221176826, + "p99": 7.683450246604985 + }, + "itl_ms": { + "mean": 7.0306507297696665, + "p50": 6.953251495132388, + "p90": 7.339486186050207, + "p95": 7.452848123203854, + "p99": 7.663379109705798 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_2_512_256.jsonl" + }, + { + "name": "c32_i1024_o1024", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 78.92528866801877, + "request_throughput": 8.108934548114409, + "input_token_throughput": 8303.548977269154, + "output_token_throughput": 1684.2003652228998, + "total_token_throughput": 9987.749342492054, + "total_input_tokens": 655360, + "total_output_tokens": 132926, + "e2e_ms": { + "mean": 3611.2669636257124, + "p50": 3619.234718993539, + "p90": 4621.976903395262, + "p95": 4919.6922949748105, + "p99": 5510.542936187703 + }, + "ttft_ms": { + "mean": 154.9310551518829, + "p50": 124.08699502702802, + "p90": 187.1325402869843, + "p95": 217.27178525179625, + "p99": 815.3002050484065 + }, + "tpot_ms": { + "mean": 16.71025031725905, + "p50": 16.85418865653697, + "p90": 18.232228898781912, + "p95": 18.607247767835663, + "p99": 18.94468969722329 + }, + "itl_ms": { + "mean": 16.631025661297535, + "p50": 16.77956095549478, + "p90": 18.166693189224386, + "p95": 18.49548590061078, + "p99": 18.88842907765743 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_1024.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 46.467179456027225, + "request_throughput": 13.773162208083754, + "input_token_throughput": 14103.718101077764, + "output_token_throughput": 1762.90020093687, + "total_token_throughput": 15866.618302014635, + "total_input_tokens": 655360, + "total_output_tokens": 81917, + "e2e_ms": { + "mean": 2110.8549089421103, + "p50": 2038.8367799750995, + "p90": 2352.120074670529, + "p95": 2431.166996533284, + "p99": 2650.418162163114 + }, + "ttft_ms": { + "mean": 408.25481356669115, + "p50": 411.61647700937465, + "p90": 687.9995263705496, + "p95": 714.2438566574128, + "p99": 757.9774171666941 + }, + "tpot_ms": { + "mean": 13.406730838620868, + "p50": 13.020844838589312, + "p90": 16.672213566187022, + "p95": 16.994947842808116, + "p99": 17.830467695905643 + }, + "itl_ms": { + "mean": 13.40621911013612, + "p50": 13.020491358131082, + "p90": 16.67184409053857, + "p95": 16.994598978927808, + "p99": 17.830109219928456 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_128.jsonl" + }, + { + "name": "c32_i1024_o2048", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 80.99940589198377, + "request_throughput": 7.901292521249697, + "input_token_throughput": 8090.92354175969, + "output_token_throughput": 1622.2094292299496, + "total_token_throughput": 9713.13297098964, + "total_input_tokens": 655360, + "total_output_tokens": 131398, + "e2e_ms": { + "mean": 3730.4086324381387, + "p50": 3610.2917045063805, + "p90": 4984.9803586083, + "p95": 5542.758034938015, + "p99": 6244.9522893788535 + }, + "ttft_ms": { + "mean": 162.95093974731572, + "p50": 125.41421098285355, + "p90": 200.1870819774922, + "p95": 334.3690510489973, + "p99": 753.3516078814864 + }, + "tpot_ms": { + "mean": 17.450724473652407, + "p50": 16.894134123043607, + "p90": 21.074347952556618, + "p95": 23.66249520612531, + "p99": 25.289011038658256 + }, + "itl_ms": { + "mean": 17.354704705635633, + "p50": 16.8038549883253, + "p90": 21.035786438586687, + "p95": 23.611833658857822, + "p99": 25.230846912060745 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_2048.jsonl" + }, + { + "name": "c32_i1024_o256", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 76.78547257801984, + "request_throughput": 8.334909957735972, + "input_token_throughput": 8534.947796721635, + "output_token_throughput": 1672.1131704899258, + "total_token_throughput": 10207.060967211562, + "total_input_tokens": 655360, + "total_output_tokens": 128394, + "e2e_ms": { + "mean": 3544.9825747800787, + "p50": 3542.4157789966557, + "p90": 4553.223528020317, + "p95": 4746.499084433889, + "p99": 4887.241407678812 + }, + "ttft_ms": { + "mean": 154.72375358949648, + "p50": 124.83429000712931, + "p90": 188.23812043992803, + "p95": 212.0150006754553, + "p99": 753.6206560855499 + }, + "tpot_ms": { + "mean": 16.974057864604074, + "p50": 16.998088166373815, + "p90": 18.762897855757615, + "p95": 19.11395051412219, + "p99": 20.002592625161597 + }, + "itl_ms": { + "mean": 16.892495206732864, + "p50": 16.91040098053608, + "p90": 18.651316667795527, + "p95": 19.051608312570696, + "p99": 19.9012634039451 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_256.jsonl" + }, + { + "name": "c32_i1024_o4096", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 77.84559064498171, + "request_throughput": 8.22140335370758, + "input_token_throughput": 8418.717034196561, + "output_token_throughput": 1664.7827953548242, + "total_token_throughput": 10083.499829551385, + "total_input_tokens": 655360, + "total_output_tokens": 129596, + "e2e_ms": { + "mean": 3549.6161134195972, + "p50": 3534.699954005191, + "p90": 4596.044332935708, + "p95": 4954.169827623991, + "p99": 5479.05965938 + }, + "ttft_ms": { + "mean": 154.27825091619525, + "p50": 124.07905000145547, + "p90": 187.61730520636777, + "p95": 214.18526259658387, + "p99": 756.5299757034518 + }, + "tpot_ms": { + "mean": 16.84736228836508, + "p50": 16.95200728817329, + "p90": 18.559453546984084, + "p95": 18.819003744342538, + "p99": 19.489634361999443 + }, + "itl_ms": { + "mean": 16.756890267590723, + "p50": 16.850854466492688, + "p90": 18.494411360209956, + "p95": 18.739564808078377, + "p99": 19.43113616649967 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_4096.jsonl" + }, + { + "name": "c32_i1024_o512", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 77.73065714200493, + "request_throughput": 8.233559621537664, + "input_token_throughput": 8431.165052454568, + "output_token_throughput": 1700.5130904595178, + "total_token_throughput": 10131.678142914085, + "total_input_tokens": 655360, + "total_output_tokens": 132182, + "e2e_ms": { + "mean": 3574.121013048716, + "p50": 3592.20840150374, + "p90": 4519.8065511882305, + "p95": 4831.873196238303, + "p99": 5478.8229196867915 + }, + "ttft_ms": { + "mean": 155.16665441473378, + "p50": 124.03260401333682, + "p90": 194.21878742869012, + "p95": 216.89559183141677, + "p99": 751.8507089995546 + }, + "tpot_ms": { + "mean": 16.619718364896162, + "p50": 16.83363489900625, + "p90": 17.92848697055871, + "p95": 18.255712662708305, + "p99": 18.667389454301812 + }, + "itl_ms": { + "mean": 16.531331980245334, + "p50": 16.762246546481126, + "p90": 17.831700172261954, + "p95": 18.077975309344417, + "p99": 18.589417282906002 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_1024_512.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 277.9606838519685, + "request_throughput": 2.3024839021508527, + "input_token_throughput": 37723.89625283957, + "output_token_throughput": 563.7559881794422, + "total_token_throughput": 38287.652241019015, + "total_input_tokens": 10485760, + "total_output_tokens": 156702, + "e2e_ms": { + "mean": 13073.635097725128, + "p50": 11227.634843002306, + "p90": 21339.115100027993, + "p95": 26643.75307828013, + "p99": 35597.739919722095 + }, + "ttft_ms": { + "mean": 1266.2669496049602, + "p50": 1001.5671220025979, + "p90": 1585.4935965267941, + "p95": 1984.2542961676368, + "p99": 8777.01818966947 + }, + "tpot_ms": { + "mean": 48.754423819978236, + "p50": 49.38631691431912, + "p90": 57.081412245865614, + "p95": 60.639865760342666, + "p99": 65.84542761235676 + }, + "itl_ms": { + "mean": 48.5449412439343, + "p50": 49.1246242388455, + "p90": 56.97187665833761, + "p95": 60.38546719900048, + "p99": 65.76520693153428 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 240.93721615005052, + "request_throughput": 2.6562936611728003, + "input_token_throughput": 43520.71534465516, + "output_token_throughput": 338.9015665771935, + "total_token_throughput": 43859.61691123235, + "total_input_tokens": 10485760, + "total_output_tokens": 81654, + "e2e_ms": { + "mean": 11539.012484155683, + "p50": 11515.057524986332, + "p90": 12479.302866500802, + "p95": 13150.691687557264, + "p99": 16925.014838652576 + }, + "ttft_ms": { + "mean": 3738.207139007227, + "p50": 4188.889887009282, + "p90": 4799.360683088889, + "p95": 4848.253755184123, + "p99": 8794.30169423169 + }, + "tpot_ms": { + "mean": 61.61473982648905, + "p50": 60.693794508032234, + "p90": 77.17945939071997, + "p95": 81.93347362540644, + "p99": 82.76267311701956 + }, + "itl_ms": { + "mean": 61.610156660920815, + "p50": 60.69343296038578, + "p90": 77.17903028975614, + "p95": 81.93312264109437, + "p99": 82.7623171365364 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_128.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 277.4810777360108, + "request_throughput": 2.3064635802261138, + "input_token_throughput": 37789.09929842465, + "output_token_throughput": 576.9366376481537, + "total_token_throughput": 38366.0359360728, + "total_input_tokens": 10485760, + "total_output_tokens": 160089, + "e2e_ms": { + "mean": 13129.881116174965, + "p50": 11338.306193007156, + "p90": 21268.143240688376, + "p95": 24681.926308575195, + "p99": 38211.25337462759 + }, + "ttft_ms": { + "mean": 1268.9881122932093, + "p50": 1007.1230739995372, + "p90": 1518.6964344407902, + "p95": 2250.822956187766, + "p99": 8796.36670469772 + }, + "tpot_ms": { + "mean": 47.56570146384879, + "p50": 48.1970007282515, + "p90": 56.213821805788776, + "p95": 58.27852197460004, + "p99": 64.82869783185714 + }, + "itl_ms": { + "mean": 47.31921304344923, + "p50": 48.01725076262202, + "p90": 55.902028920966025, + "p95": 57.776933802019016, + "p99": 64.19539631448289 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 264.69584388204385, + "request_throughput": 2.4178694709131987, + "input_token_throughput": 39614.37341144185, + "output_token_throughput": 508.6630678644125, + "total_token_throughput": 40123.03647930626, + "total_input_tokens": 10485760, + "total_output_tokens": 134641, + "e2e_ms": { + "mean": 12624.999633311927, + "p50": 12678.140514501138, + "p90": 15806.112510379171, + "p95": 16418.681864690734, + "p99": 19908.81419315061 + }, + "ttft_ms": { + "mean": 1298.0343049251132, + "p50": 1084.8720205249265, + "p90": 1652.756949950708, + "p95": 1994.6635739383053, + "p99": 8761.715852337074 + }, + "tpot_ms": { + "mean": 54.06142052439333, + "p50": 54.97810719405666, + "p90": 62.148850621784206, + "p95": 64.54428565345222, + "p99": 70.66540055128465 + }, + "itl_ms": { + "mean": 53.84442161161199, + "p50": 54.805519818907335, + "p90": 62.00671392397239, + "p95": 64.18330302536899, + "p99": 68.9827079291856 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_256.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 275.93009687698213, + "request_throughput": 2.3194280263139655, + "input_token_throughput": 38001.50878312801, + "output_token_throughput": 561.1384975848795, + "total_token_throughput": 38562.647280712896, + "total_input_tokens": 10485760, + "total_output_tokens": 154835, + "e2e_ms": { + "mean": 13034.17151498952, + "p50": 11792.888013500487, + "p90": 20832.816231111065, + "p95": 25370.34360906109, + "p99": 27540.55410957255 + }, + "ttft_ms": { + "mean": 1268.6738726532894, + "p50": 1003.3367375144735, + "p90": 1603.3396552025806, + "p95": 1990.857944494928, + "p99": 8787.033058899106 + }, + "tpot_ms": { + "mean": 48.88744400726067, + "p50": 49.624181990480324, + "p90": 57.32051524431882, + "p95": 60.153738932365926, + "p99": 66.01202965773221 + }, + "itl_ms": { + "mean": 48.621612374564776, + "p50": 49.502786324277764, + "p90": 57.18351559129017, + "p95": 59.71803436088074, + "p99": 65.83749939502626 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_16384_512.jsonl" + }, + { + "name": "c32_i2048_o1024", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 87.94604192103725, + "request_throughput": 7.277189354065836, + "input_token_throughput": 14903.683797126832, + "output_token_throughput": 1574.8519998700413, + "total_token_throughput": 16478.53579699687, + "total_input_tokens": 1310720, + "total_output_tokens": 138502, + "e2e_ms": { + "mean": 4023.455450163783, + "p50": 3788.9970429823734, + "p90": 5616.974991484312, + "p95": 6506.863309702021, + "p99": 8623.713361859554 + }, + "ttft_ms": { + "mean": 200.89388552732999, + "p50": 151.6667715040967, + "p90": 236.44375635194592, + "p95": 270.092881628077, + "p99": 1352.101640147157 + }, + "tpot_ms": { + "mean": 17.77137965460594, + "p50": 17.871603376209883, + "p90": 19.549240949504515, + "p95": 19.912648025465035, + "p99": 20.514297965166563 + }, + "itl_ms": { + "mean": 17.680424030826366, + "p50": 17.754758553165367, + "p90": 19.45759664143349, + "p95": 19.809925328626218, + "p99": 20.429382566472793 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_1024.jsonl" + }, + { + "name": "c32_i2048_o128", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 59.23938118299702, + "request_throughput": 10.803623995040883, + "input_token_throughput": 22125.82194184373, + "output_token_throughput": 1375.94279974341, + "total_token_throughput": 23501.76474158714, + "total_input_tokens": 1310720, + "total_output_tokens": 81510, + "e2e_ms": { + "mean": 2732.466652070889, + "p50": 2755.088367994176, + "p90": 3012.18083424028, + "p95": 3094.9215329048457, + "p99": 3404.7346471465426 + }, + "ttft_ms": { + "mean": 367.5306692531194, + "p50": 363.3325230330229, + "p90": 541.3078215962743, + "p95": 672.5567041052273, + "p99": 1352.0049781136913 + }, + "tpot_ms": { + "mean": 18.711008783973277, + "p50": 19.23976635411895, + "p90": 20.943899588904678, + "p95": 21.551382261961805, + "p99": 22.092666742765992 + }, + "itl_ms": { + "mean": 18.69443736023786, + "p50": 19.23653134626087, + "p90": 20.932656177273753, + "p95": 21.551018130739116, + "p99": 22.078695479467857 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_128.jsonl" + }, + { + "name": "c32_i2048_o2048", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 89.13132148201112, + "request_throughput": 7.180416371692275, + "input_token_throughput": 14705.49272922578, + "output_token_throughput": 1556.097202350943, + "total_token_throughput": 16261.589931576722, + "total_input_tokens": 1310720, + "total_output_tokens": 138697, + "e2e_ms": { + "mean": 4056.525911160861, + "p50": 3742.62464849744, + "p90": 5721.507992275293, + "p95": 6825.735677420744, + "p99": 9339.784606644538 + }, + "ttft_ms": { + "mean": 199.91363017315962, + "p50": 151.93368299514987, + "p90": 230.94767580041662, + "p95": 291.09600226802223, + "p99": 1361.131886652438 + }, + "tpot_ms": { + "mean": 17.872137121852198, + "p50": 17.839739867944665, + "p90": 20.02253391370516, + "p95": 20.276428343314567, + "p99": 20.978136194178266 + }, + "itl_ms": { + "mean": 17.781535712600615, + "p50": 17.74159216290751, + "p90": 19.928170335933803, + "p95": 20.192582616625344, + "p99": 20.88261418152327 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_2048.jsonl" + }, + { + "name": "c32_i2048_o256", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 83.4020835319534, + "request_throughput": 7.673669204616456, + "input_token_throughput": 15715.674531054501, + "output_token_throughput": 1544.122095590814, + "total_token_throughput": 17259.796626645315, + "total_input_tokens": 1310720, + "total_output_tokens": 128783, + "e2e_ms": { + "mean": 3838.3926361287195, + "p50": 3821.959828987019, + "p90": 4955.226511094952, + "p95": 5110.333283661748, + "p99": 5339.235162911937 + }, + "ttft_ms": { + "mean": 202.6343736236413, + "p50": 151.54230897314847, + "p90": 241.23692908906378, + "p95": 380.56901601084917, + "p99": 1355.983304544934 + }, + "tpot_ms": { + "mean": 18.190302456605295, + "p50": 18.424039937799243, + "p90": 19.81137767760755, + "p95": 20.183518817431427, + "p99": 20.729456373731534 + }, + "itl_ms": { + "mean": 18.11254713064598, + "p50": 18.35936738325809, + "p90": 19.743566255628185, + "p95": 20.059024347520136, + "p99": 20.662909090368444 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_256.jsonl" + }, + { + "name": "c32_i2048_o4096", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 88.69394837901928, + "request_throughput": 7.215824886553289, + "input_token_throughput": 14778.009367661136, + "output_token_throughput": 1520.4532266814747, + "total_token_throughput": 16298.462594342609, + "total_input_tokens": 1310720, + "total_output_tokens": 134855, + "e2e_ms": { + "mean": 3982.980794719242, + "p50": 3718.501230527181, + "p90": 5520.199165504891, + "p95": 6383.295495348279, + "p99": 9405.390494499596 + }, + "ttft_ms": { + "mean": 199.18991788063067, + "p50": 151.69027147931047, + "p90": 225.76481199939738, + "p95": 288.1128885026552, + "p99": 1360.760756857344 + }, + "tpot_ms": { + "mean": 18.058210034347876, + "p50": 18.061947073373684, + "p90": 19.997909734484665, + "p95": 20.40678994044308, + "p99": 20.835222154193175 + }, + "itl_ms": { + "mean": 17.954757087634338, + "p50": 17.966942533526677, + "p90": 19.907732744673698, + "p95": 20.27250962252495, + "p99": 20.742511472732147 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_4096.jsonl" + }, + { + "name": "c32_i2048_o512", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 90.84299184399424, + "request_throughput": 7.045122436071675, + "input_token_throughput": 14428.41074907479, + "output_token_throughput": 1530.3216811566379, + "total_token_throughput": 15958.732430231428, + "total_input_tokens": 1310720, + "total_output_tokens": 139019, + "e2e_ms": { + "mean": 4060.401959133651, + "p50": 3761.1486914684065, + "p90": 5925.088173424592, + "p95": 6700.475931522667, + "p99": 9080.734873839541 + }, + "ttft_ms": { + "mean": 197.1673974629084, + "p50": 151.31841399124824, + "p90": 224.7515225026291, + "p95": 253.29808431270044, + "p99": 1349.4623282918474 + }, + "tpot_ms": { + "mean": 17.877858826093764, + "p50": 17.956536891848835, + "p90": 19.937387328063277, + "p95": 20.253823827590228, + "p99": 21.01890637646956 + }, + "itl_ms": { + "mean": 17.792685269469718, + "p50": 17.879439161587523, + "p90": 19.834828564609182, + "p95": 20.16044198291448, + "p99": 20.8988025100466 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_2048_512.jsonl" + }, + { + "name": "c32_i4096_o1024", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 113.81290011497913, + "request_throughput": 5.62326414100196, + "input_token_throughput": 23032.88992154403, + "output_token_throughput": 1212.692057407954, + "total_token_throughput": 24245.58197895198, + "total_input_tokens": 2621440, + "total_output_tokens": 138020, + "e2e_ms": { + "mean": 5215.721413190931, + "p50": 4851.276391505962, + "p90": 7216.958626796259, + "p95": 8904.339508028348, + "p99": 13457.617856655856 + }, + "ttft_ms": { + "mean": 339.673375241091, + "p50": 247.79371902695857, + "p90": 392.1883804141544, + "p95": 542.3683993751182, + "p99": 2422.6248358923476 + }, + "tpot_ms": { + "mean": 22.719399348508016, + "p50": 22.914273851038725, + "p90": 25.531362948015392, + "p95": 26.101945052613324, + "p99": 27.20651141262886 + }, + "itl_ms": { + "mean": 22.60461888887343, + "p50": 22.75339210173309, + "p90": 25.422672764186185, + "p95": 25.91771771805469, + "p99": 27.09015545979034 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_1024.jsonl" + }, + { + "name": "c32_i4096_o128", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 85.65506197494688, + "request_throughput": 7.471829279479042, + "input_token_throughput": 30604.612728746157, + "output_token_throughput": 950.8719989464524, + "total_token_throughput": 31555.48472769261, + "total_input_tokens": 2621440, + "total_output_tokens": 81447, + "e2e_ms": { + "mean": 4002.7959108847426, + "p50": 4027.488345018355, + "p90": 4394.019550387748, + "p95": 4613.657232854166, + "p99": 5507.045013857423 + }, + "ttft_ms": { + "mean": 556.1845627042203, + "p50": 511.0022419830784, + "p90": 1002.5600348075387, + "p95": 1180.2316012704969, + "p99": 2449.125620237319 + }, + "tpot_ms": { + "mean": 27.29475949831838, + "p50": 27.766935429013312, + "p90": 30.37622396083811, + "p95": 31.52356485168882, + "p99": 34.81999862995494 + }, + "itl_ms": { + "mean": 27.283351684511892, + "p50": 27.752212318866768, + "p90": 30.37272536720142, + "p95": 31.523207846206727, + "p99": 34.81966031348612 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_128.jsonl" + }, + { + "name": "c32_i4096_o2048", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.274240713974, + "request_throughput": 5.600562261462812, + "input_token_throughput": 22939.90302295168, + "output_token_throughput": 1189.489417306433, + "total_token_throughput": 24129.392440258114, + "total_input_tokens": 2621440, + "total_output_tokens": 135928, + "e2e_ms": { + "mean": 5165.783436226775, + "p50": 4726.68347551371, + "p90": 7188.446018210382, + "p95": 9243.12443975359, + "p99": 13018.90308270814 + }, + "ttft_ms": { + "mean": 341.10774283180945, + "p50": 247.46722800773568, + "p90": 392.88402279489674, + "p95": 550.6020500150042, + "p99": 2435.4554109059973 + }, + "tpot_ms": { + "mean": 22.783776407589954, + "p50": 23.01332679174569, + "p90": 25.46372044856561, + "p95": 26.016316538121043, + "p99": 27.730684264550362 + }, + "itl_ms": { + "mean": 22.657550920079434, + "p50": 22.83503046063882, + "p90": 25.269074608539984, + "p95": 25.845271448447257, + "p99": 27.485328640230005 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_2048.jsonl" + }, + { + "name": "c32_i4096_o256", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 106.208948674961, + "request_throughput": 6.025857594717737, + "input_token_throughput": 24681.91270796385, + "output_token_throughput": 1178.6106685143307, + "total_token_throughput": 25860.52337647818, + "total_input_tokens": 2621440, + "total_output_tokens": 125179, + "e2e_ms": { + "mean": 4937.5405230529395, + "p50": 4875.018366990844, + "p90": 6434.248026396381, + "p95": 6734.163024299778, + "p99": 7405.486856222272 + }, + "ttft_ms": { + "mean": 350.7942819734126, + "p50": 248.04657051572576, + "p90": 422.6160959864501, + "p95": 564.3712449935265, + "p99": 2447.9905850067735 + }, + "tpot_ms": { + "mean": 23.550120668178824, + "p50": 23.852025049090017, + "p90": 25.96803448657071, + "p95": 26.71366147238315, + "p99": 28.321813953149746 + }, + "itl_ms": { + "mean": 23.433249650000544, + "p50": 23.708379496861397, + "p90": 25.880524181822437, + "p95": 26.616908152126097, + "p99": 28.186099664674693 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_256.jsonl" + }, + { + "name": "c32_i4096_o4096", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.9426989480271, + "request_throughput": 5.567991754651461, + "input_token_throughput": 22806.494227052382, + "output_token_throughput": 1192.220134503394, + "total_token_throughput": 23998.714361555776, + "total_input_tokens": 2621440, + "total_output_tokens": 137037, + "e2e_ms": { + "mean": 5203.629682052178, + "p50": 4792.196563503239, + "p90": 7185.140736214817, + "p95": 8545.926395323471, + "p99": 12625.456661586657 + }, + "ttft_ms": { + "mean": 341.8584196219854, + "p50": 247.48384198755957, + "p90": 401.0035815590527, + "p95": 550.5504803440999, + "p99": 2448.394843073911 + }, + "tpot_ms": { + "mean": 22.82941362215177, + "p50": 23.063525754888257, + "p90": 25.525615854479813, + "p95": 26.158675644158695, + "p99": 27.229142202418178 + }, + "itl_ms": { + "mean": 22.6960209124615, + "p50": 22.959077084300404, + "p90": 25.33381873888988, + "p95": 26.059027535414014, + "p99": 27.1434838141601 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_4096.jsonl" + }, + { + "name": "c32_i4096_o512", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 112.19549296301557, + "request_throughput": 5.7043289627594165, + "input_token_throughput": 23364.93143146257, + "output_token_throughput": 1200.3958130866804, + "total_token_throughput": 24565.32724454925, + "total_input_tokens": 2621440, + "total_output_tokens": 134679, + "e2e_ms": { + "mean": 5150.640929284236, + "p50": 4812.448643002426, + "p90": 7110.52369940444, + "p95": 8529.089289769758, + "p99": 12164.940070008744 + }, + "ttft_ms": { + "mean": 340.2039304295613, + "p50": 247.1121745184064, + "p90": 393.92301979823975, + "p95": 560.246634390205, + "p99": 2385.055927421201 + }, + "tpot_ms": { + "mean": 22.994319271082635, + "p50": 23.234810646082217, + "p90": 25.56160844078038, + "p95": 26.289083146584108, + "p99": 27.726785424863163 + }, + "itl_ms": { + "mean": 22.878491774019594, + "p50": 23.078625202867357, + "p90": 25.48089463753205, + "p95": 26.23845005583209, + "p99": 27.65327865640478 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_4096_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 78.27041419100715, + "request_throughput": 8.17678054492182, + "input_token_throughput": 4186.511638999972, + "output_token_throughput": 1690.8815593722238, + "total_token_throughput": 5877.393198372196, + "total_input_tokens": 327680, + "total_output_tokens": 132346, + "e2e_ms": { + "mean": 3610.3891835380637, + "p50": 3672.895397496177, + "p90": 4614.467557013268, + "p95": 4786.530874358141, + "p99": 5755.867604461382 + }, + "ttft_ms": { + "mean": 148.9999322566291, + "p50": 122.67996551236138, + "p90": 199.04762358637527, + "p95": 289.9259683268591, + "p99": 473.49165664054453 + }, + "tpot_ms": { + "mean": 16.81089925890317, + "p50": 16.88569357827005, + "p90": 18.596486083093502, + "p95": 19.2741748259838, + "p99": 22.35552161032398 + }, + "itl_ms": { + "mean": 16.743994123153765, + "p50": 16.83361651378548, + "p90": 18.490313803081033, + "p95": 19.17608061977105, + "p99": 22.35535027858728 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_512_256.jsonl" + }, + { + "name": "c32_i65536_o1024", + "config": { + "concurrency": 32, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 266.5209177720244, + "request_throughput": 0.6003281143465826, + "input_token_throughput": 39343.10330181764, + "output_token_throughput": 131.31427091188561, + "total_token_throughput": 39474.41757272952, + "total_input_tokens": 10485760, + "total_output_tokens": 34998, + "e2e_ms": { + "mean": 46303.99489910378, + "p50": 40970.881291490514, + "p90": 78206.72828421228, + "p95": 103438.82800290309, + "p99": 127750.74539481307 + }, + "ttft_ms": { + "mean": 8656.403181048518, + "p50": 5203.471615008311, + "p90": 22916.200463398112, + "p95": 33860.81311377059, + "p99": 43057.064939973636 + }, + "tpot_ms": { + "mean": 171.13581258550425, + "p50": 177.9552146543067, + "p90": 228.0412552804808, + "p95": 244.27115008225184, + "p99": 281.95110766554944 + }, + "itl_ms": { + "mean": 170.32872628814005, + "p50": 176.05291277692726, + "p90": 227.71504553612323, + "p95": 243.7796027544607, + "p99": 281.8399921541697 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_32_65536_1024.jsonl" + }, + { + "name": "c4_i512_o256", + "config": { + "concurrency": 4, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "metrics": { + "success": 80, + "failed": 0, + "duration_s": 43.39664837101009, + "request_throughput": 1.843460336292739, + "input_token_throughput": 943.8516921818824, + "output_token_throughput": 379.1767479212128, + "total_token_throughput": 1323.0284401030951, + "total_input_tokens": 40960, + "total_output_tokens": 16455, + "e2e_ms": { + "mean": 1720.266553755937, + "p50": 1804.6963794913609, + "p90": 2158.3282727748156, + "p95": 2165.1579623343423, + "p99": 2253.321928069927 + }, + "ttft_ms": { + "mean": 118.070524858922, + "p50": 114.08401752123609, + "p90": 120.62304747523739, + "p95": 125.54392934544008, + "p99": 241.45657837681935 + }, + "tpot_ms": { + "mean": 7.8335008228886505, + "p50": 7.83555123474636, + "p90": 8.170451865484264, + "p95": 8.290225660170474, + "p99": 8.677201510073528 + }, + "itl_ms": { + "mean": 7.794213050172942, + "p50": 7.793371932429836, + "p90": 8.115348610206611, + "p95": 8.234429535969335, + "p99": 8.634192133473178 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_4_512_256.jsonl" + }, + { + "name": "c64_i512_o256", + "config": { + "concurrency": 64, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 1280 + }, + "metrics": { + "success": 1280, + "failed": 0, + "duration_s": 109.33125554397702, + "request_throughput": 11.707539565253938, + "input_token_throughput": 5994.2602574100165, + "output_token_throughput": 2402.359679244252, + "total_token_throughput": 8396.619936654268, + "total_input_tokens": 655360, + "total_output_tokens": 262653, + "e2e_ms": { + "mean": 5151.599965186915, + "p50": 5233.160602510907, + "p90": 6570.279945392394, + "p95": 6720.288137154421, + "p99": 6878.420427516685 + }, + "ttft_ms": { + "mean": 171.44839711986606, + "p50": 132.15252198278904, + "p90": 210.84870708873493, + "p95": 299.06158958619926, + "p99": 771.50551313709 + }, + "tpot_ms": { + "mean": 24.389172122073912, + "p50": 24.77321306780196, + "p90": 26.27769079484252, + "p95": 26.718942784647773, + "p99": 27.576043138567123 + }, + "itl_ms": { + "mean": 24.284871248079536, + "p50": 24.67090953529502, + "p90": 26.175376179470668, + "p95": 26.56671499095787, + "p99": 27.389117637935758 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_64_512_256.jsonl" + }, + { + "name": "c8_i1024_o1024", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.79919548804173, + "request_throughput": 3.3473366730623813, + "input_token_throughput": 3427.6727532158784, + "output_token_throughput": 696.0995820175289, + "total_token_throughput": 4123.772335233407, + "total_input_tokens": 163840, + "total_output_tokens": 33273, + "e2e_ms": { + "mean": 2052.0071145107067, + "p50": 2064.3975179700647, + "p90": 2597.689549595816, + "p95": 2788.772759854328, + "p99": 3052.1448046178557 + }, + "ttft_ms": { + "mean": 129.12154367477342, + "p50": 122.31972347944975, + "p90": 124.8896159871947, + "p95": 192.22247132274794, + "p99": 281.036811490776 + }, + "tpot_ms": { + "mean": 9.273661335882394, + "p50": 9.290390007233619, + "p90": 9.797906786052058, + "p95": 9.954364221454668, + "p99": 10.238029201305554 + }, + "itl_ms": { + "mean": 9.22141272050547, + "p50": 9.236384602551324, + "p90": 9.75097784670255, + "p95": 9.913460638994195, + "p99": 10.132330933263706 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_1024.jsonl" + }, + { + "name": "c8_i1024_o128", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 31.03394475398818, + "request_throughput": 5.155644932294286, + "input_token_throughput": 5279.380410669349, + "output_token_throughput": 659.9225513336686, + "total_token_throughput": 5939.3029620030175, + "total_input_tokens": 163840, + "total_output_tokens": 20480, + "e2e_ms": { + "mean": 1315.4882694318076, + "p50": 1366.9841965020169, + "p90": 1380.8117090375163, + "p95": 1387.6815518015064, + "p99": 1458.9439847879112 + }, + "ttft_ms": { + "mean": 187.55042687007517, + "p50": 205.4987929877825, + "p90": 253.20741288596764, + "p95": 269.2095974576661, + "p99": 292.6897038030438 + }, + "tpot_ms": { + "mean": 8.881400335131751, + "p50": 9.162143610260177, + "p90": 9.883002829362132, + "p95": 9.902873972781904, + "p99": 9.973658800444118 + }, + "itl_ms": { + "mean": 8.880977010622292, + "p50": 9.16174857090704, + "p90": 9.88247430792599, + "p95": 9.902345587171277, + "p99": 9.97331234991301 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_128.jsonl" + }, + { + "name": "c8_i1024_o2048", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.63014482904691, + "request_throughput": 3.359217163295819, + "input_token_throughput": 3439.838375214919, + "output_token_throughput": 684.7554236305822, + "total_token_throughput": 4124.593798845501, + "total_input_tokens": 163840, + "total_output_tokens": 32615, + "e2e_ms": { + "mean": 2064.275790168176, + "p50": 2025.2659794932697, + "p90": 2682.616906025214, + "p95": 2867.5991577445534, + "p99": 3177.5962958741 + }, + "ttft_ms": { + "mean": 139.63929469391587, + "p50": 123.93425550544634, + "p90": 188.980170188006, + "p95": 191.21294881624635, + "p99": 283.84840004029684 + }, + "tpot_ms": { + "mean": 9.488485002558638, + "p50": 9.326192293000322, + "p90": 10.394212695334996, + "p95": 10.925130294149847, + "p99": 11.765321442654237 + }, + "itl_ms": { + "mean": 9.439770830290232, + "p50": 9.27684741342917, + "p90": 10.347793105855219, + "p95": 10.861698161034582, + "p99": 11.710557326254763 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_2048.jsonl" + }, + { + "name": "c8_i1024_o256", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 47.67866635101382, + "request_throughput": 3.3557985624444346, + "input_token_throughput": 3436.337727943101, + "output_token_throughput": 684.9394603359244, + "total_token_throughput": 4121.277188279026, + "total_input_tokens": 163840, + "total_output_tokens": 32657, + "e2e_ms": { + "mean": 2050.7159092150687, + "p50": 2055.9715820127167, + "p90": 2524.6206711861305, + "p95": 2669.197603108478, + "p99": 2870.3536581806834 + }, + "ttft_ms": { + "mean": 134.21947423594247, + "p50": 122.28806095663458, + "p90": 190.7292490301188, + "p95": 195.34277862985618, + "p99": 290.7564730977174 + }, + "tpot_ms": { + "mean": 9.439803513870618, + "p50": 9.293550548206369, + "p90": 10.193549946169153, + "p95": 10.826949428327007, + "p99": 11.661622554928408 + }, + "itl_ms": { + "mean": 9.387716787724182, + "p50": 9.246371719530723, + "p90": 10.10971836153746, + "p95": 10.739477732631839, + "p99": 11.613613014141018 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_256.jsonl" + }, + { + "name": "c8_i1024_o4096", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.021476240013726, + "request_throughput": 3.3318425947655594, + "input_token_throughput": 3411.806817039933, + "output_token_throughput": 683.4233882350682, + "total_token_throughput": 4095.230205275001, + "total_input_tokens": 163840, + "total_output_tokens": 32819, + "e2e_ms": { + "mean": 2050.123825306946, + "p50": 2050.3479074977804, + "p90": 2538.145926152356, + "p95": 2647.510393260746, + "p99": 2950.3177008335465 + }, + "ttft_ms": { + "mean": 132.58988224442874, + "p50": 122.42957402486354, + "p90": 142.4261361011302, + "p95": 194.91657257894985, + "p99": 285.38757893256843 + }, + "tpot_ms": { + "mean": 9.387625075467776, + "p50": 9.359307527011389, + "p90": 10.081905747530234, + "p95": 10.450971488236782, + "p99": 11.053297382297417 + }, + "itl_ms": { + "mean": 9.337072189720585, + "p50": 9.312614422785066, + "p90": 9.942381647700445, + "p95": 10.406373012197426, + "p99": 11.00866874531187 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_4096.jsonl" + }, + { + "name": "c8_i1024_o512", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.88774840300903, + "request_throughput": 3.272803621083765, + "input_token_throughput": 3351.3509079897754, + "output_token_throughput": 699.8481443234996, + "total_token_throughput": 4051.199052313275, + "total_input_tokens": 163840, + "total_output_tokens": 34214, + "e2e_ms": { + "mean": 2097.309012806363, + "p50": 2095.9312209743075, + "p90": 2660.76910031843, + "p95": 2846.3267137063667, + "p99": 3059.683581781573 + }, + "ttft_ms": { + "mean": 128.42690491161193, + "p50": 122.46163000236265, + "p90": 125.48143291496672, + "p95": 130.8216415811331, + "p99": 286.1712126014754 + }, + "tpot_ms": { + "mean": 9.250483832974629, + "p50": 9.276897661059472, + "p90": 9.808938170245048, + "p95": 9.963068685422419, + "p99": 10.231402362643065 + }, + "itl_ms": { + "mean": 9.202953803540257, + "p50": 9.222722470294482, + "p90": 9.769427792921025, + "p95": 9.900009820334152, + "p99": 10.176064141791748 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_1024_512.jsonl" + }, + { + "name": "c8_i131072_o512", + "config": { + "concurrency": 8, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 593.70801256696, + "request_throughput": 0.26949274157211206, + "input_token_throughput": 35322.95262333987, + "output_token_throughput": 49.96900727637406, + "total_token_throughput": 35372.92163061625, + "total_input_tokens": 20971520, + "total_output_tokens": 29667, + "e2e_ms": { + "mean": 26223.813140128914, + "p50": 25749.483308027266, + "p90": 38876.46534329397, + "p95": 43791.96180334258, + "p99": 52427.09982689234 + }, + "ttft_ms": { + "mean": 8837.25882775725, + "p50": 6957.265121513046, + "p90": 12366.58897944144, + "p95": 12856.652719894191, + "p99": 21274.84452185161 + }, + "tpot_ms": { + "mean": 95.9775316206848, + "p50": 94.03290558605093, + "p90": 149.15055402178177, + "p95": 174.739404388061, + "p99": 250.30791511469616 + }, + "itl_ms": { + "mean": 95.50966075588461, + "p50": 93.91736107077944, + "p90": 148.93209375100713, + "p95": 174.60800165421526, + "p99": 249.69256104941496 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_131072_512.jsonl" + }, + { + "name": "c8_i16384_o1024", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 117.03195754898479, + "request_throughput": 1.3671479427576911, + "input_token_throughput": 22399.35189414201, + "output_token_throughput": 381.4684547279648, + "total_token_throughput": 22780.820348869976, + "total_input_tokens": 2621440, + "total_output_tokens": 44644, + "e2e_ms": { + "mean": 5043.960333102223, + "p50": 4365.808989998186, + "p90": 7655.828198097879, + "p95": 9664.448152296247, + "p99": 11942.893994341719 + }, + "ttft_ms": { + "mean": 880.5494676955277, + "p50": 782.9482170054689, + "p90": 1082.6903782086447, + "p95": 1325.487311300822, + "p99": 2519.2373008432323 + }, + "tpot_ms": { + "mean": 15.139856510889135, + "p50": 15.27841200283051, + "p90": 18.022824161834254, + "p95": 18.881136649500213, + "p99": 22.888170543028938 + }, + "itl_ms": { + "mean": 15.010650016559842, + "p50": 15.1465002130646, + "p90": 17.968179468660793, + "p95": 18.85452186357664, + "p99": 22.829326886115528 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_1024.jsonl" + }, + { + "name": "c8_i16384_o128", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 83.84433197398903, + "request_throughput": 1.9082983456727485, + "input_token_throughput": 31265.56009550231, + "output_token_throughput": 241.22083775769588, + "total_token_throughput": 31506.780933260008, + "total_input_tokens": 2621440, + "total_output_tokens": 20225, + "e2e_ms": { + "mean": 3640.27295542146, + "p50": 3633.8590744999237, + "p90": 3677.6862947270274, + "p95": 3713.6206373630557, + "p99": 5332.837568135582 + }, + "ttft_ms": { + "mean": 1766.892234862462, + "p50": 1687.3688155028503, + "p90": 2631.7813128232956, + "p95": 2637.4724537250586, + "p99": 2676.544717006618 + }, + "tpot_ms": { + "mean": 14.952028106677584, + "p50": 15.45864561090379, + "p90": 20.86819931141837, + "p95": 23.249104839698937, + "p99": 26.077070266717733 + }, + "itl_ms": { + "mean": 14.950050644874448, + "p50": 15.458261126898126, + "p90": 20.867791510470617, + "p95": 23.248591416115467, + "p99": 26.076698547013414 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_128.jsonl" + }, + { + "name": "c8_i16384_o2048", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 111.6486262770486, + "request_throughput": 1.4330673411329817, + "input_token_throughput": 23479.375317122773, + "output_token_throughput": 353.0003128045817, + "total_token_throughput": 23832.375629927355, + "total_input_tokens": 2621440, + "total_output_tokens": 39412, + "e2e_ms": { + "mean": 4778.323991763318, + "p50": 4169.615545513807, + "p90": 7071.359802276127, + "p95": 8910.118855611656, + "p99": 13501.66908308048 + }, + "ttft_ms": { + "mean": 858.3292873940081, + "p50": 782.5817215198185, + "p90": 952.6779550826175, + "p95": 1105.0728265632642, + "p99": 2515.795475198537 + }, + "tpot_ms": { + "mean": 15.798579513726072, + "p50": 16.022473646922144, + "p90": 18.945479346672712, + "p95": 19.890483962991468, + "p99": 21.084441579797186 + }, + "itl_ms": { + "mean": 15.70931575959753, + "p50": 15.979023863701155, + "p90": 18.913301621034854, + "p95": 19.76568012612783, + "p99": 21.016233379075533 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_2048.jsonl" + }, + { + "name": "c8_i16384_o256", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 101.76510345499264, + "request_throughput": 1.572248192827345, + "input_token_throughput": 25759.71439128322, + "output_token_throughput": 329.16981227081504, + "total_token_throughput": 26088.88420355404, + "total_input_tokens": 2621440, + "total_output_tokens": 33498, + "e2e_ms": { + "mean": 4427.855411094424, + "p50": 4454.245780507335, + "p90": 5497.165809822036, + "p95": 5958.67853332893, + "p99": 6667.5567474006675 + }, + "ttft_ms": { + "mean": 867.7743797547009, + "p50": 781.0635255009402, + "p90": 1016.5140870318279, + "p95": 1267.3269381921277, + "p99": 2511.917921541025 + }, + "tpot_ms": { + "mean": 17.06346602830906, + "p50": 17.093144427779265, + "p90": 21.41804815869456, + "p95": 21.934133880069854, + "p99": 23.133364661873244 + }, + "itl_ms": { + "mean": 16.998988221217065, + "p50": 17.06275551950803, + "p90": 21.301843049957412, + "p95": 21.847790275915557, + "p99": 23.01043882594314 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_256.jsonl" + }, + { + "name": "c8_i16384_o512", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 109.64312597998651, + "request_throughput": 1.4592798095633035, + "input_token_throughput": 23908.840399885165, + "output_token_throughput": 355.69033308224544, + "total_token_throughput": 24264.53073296741, + "total_input_tokens": 2621440, + "total_output_tokens": 38999, + "e2e_ms": { + "mean": 4755.6579099466035, + "p50": 4298.210411972832, + "p90": 7776.077105075818, + "p95": 8684.670029362314, + "p99": 10122.991236373782 + }, + "ttft_ms": { + "mean": 908.3466928703274, + "p50": 778.5314369830303, + "p90": 1214.549721876392, + "p95": 1361.7619198863397, + "p99": 2515.331039446754 + }, + "tpot_ms": { + "mean": 15.817801648689121, + "p50": 16.17100685840291, + "p90": 19.34598438045577, + "p95": 20.351946538990223, + "p99": 22.15198565504161 + }, + "itl_ms": { + "mean": 15.714668041430865, + "p50": 15.998185230128716, + "p90": 19.235813678199687, + "p95": 20.317084004124045, + "p99": 22.094216228598047 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_16384_512.jsonl" + }, + { + "name": "c8_i2048_o1024", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.1054894879926, + "request_throughput": 3.0128712030076805, + "input_token_throughput": 6170.36022375973, + "output_token_throughput": 677.8395293416718, + "total_token_throughput": 6848.199753101401, + "total_input_tokens": 327680, + "total_output_tokens": 35997, + "e2e_ms": { + "mean": 2286.0924268738017, + "p50": 2064.6599415049423, + "p90": 3458.0778610950797, + "p95": 3870.0411044963403, + "p99": 5178.540953937917 + }, + "ttft_ms": { + "mean": 150.15353084418166, + "p50": 138.43873603036627, + "p90": 145.44060388579965, + "p95": 153.03919319703712, + "p99": 410.55034047400113 + }, + "tpot_ms": { + "mean": 9.488409542087934, + "p50": 9.50819875088937, + "p90": 10.10307734654145, + "p95": 10.240756052118074, + "p99": 10.652978840145037 + }, + "itl_ms": { + "mean": 9.435480404556735, + "p50": 9.475025016652458, + "p90": 10.058684681432286, + "p95": 10.183135112300214, + "p99": 10.599880392631377 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_1024.jsonl" + }, + { + "name": "c8_i2048_o128", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 33.34751936199609, + "request_throughput": 4.79795808087426, + "input_token_throughput": 9826.218149630484, + "output_token_throughput": 612.3993745475884, + "total_token_throughput": 10438.617524178073, + "total_input_tokens": 327680, + "total_output_tokens": 20422, + "e2e_ms": { + "mean": 1411.2315205631603, + "p50": 1428.1102574896067, + "p90": 1452.4172595120035, + "p95": 1530.949007422896, + "p99": 1595.8804498135578 + }, + "ttft_ms": { + "mean": 228.93475683886209, + "p50": 150.7378289825283, + "p90": 379.75782579160295, + "p95": 388.7717862293357, + "p99": 404.45577931881417 + }, + "tpot_ms": { + "mean": 9.333638432390616, + "p50": 9.567953153480373, + "p90": 10.250220458961905, + "p95": 10.326194063064598, + "p99": 11.017481187701328 + }, + "itl_ms": { + "mean": 9.331251454181302, + "p50": 9.567510058890706, + "p90": 10.249756787184948, + "p95": 10.32570392511213, + "p99": 11.017141694314851 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_128.jsonl" + }, + { + "name": "c8_i2048_o2048", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 51.12094455503393, + "request_throughput": 3.1298326232558753, + "input_token_throughput": 6409.897212428033, + "output_token_throughput": 672.6988380071643, + "total_token_throughput": 7082.596050435197, + "total_input_tokens": 327680, + "total_output_tokens": 34389, + "e2e_ms": { + "mean": 2202.4928271144745, + "p50": 2062.3054489842616, + "p90": 3023.14127808786, + "p95": 3406.000318186131, + "p99": 4277.616084612091 + }, + "ttft_ms": { + "mean": 150.70150687461137, + "p50": 138.49432798451744, + "p90": 144.4377199979499, + "p95": 165.44266314012995, + "p99": 417.99726528290194 + }, + "tpot_ms": { + "mean": 9.560429413090715, + "p50": 9.588894799229887, + "p90": 10.169284756728882, + "p95": 10.25023479426485, + "p99": 10.675569194381492 + }, + "itl_ms": { + "mean": 9.507038628210164, + "p50": 9.556331128852783, + "p90": 10.125696983636297, + "p95": 10.207410243990491, + "p99": 10.55116458914923 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_2048.jsonl" + }, + { + "name": "c8_i2048_o256", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.032057252014056, + "request_throughput": 3.3311086210718357, + "input_token_throughput": 6822.1104559551195, + "output_token_throughput": 658.8516463902457, + "total_token_throughput": 7480.962102345365, + "total_input_tokens": 327680, + "total_output_tokens": 31646, + "e2e_ms": { + "mean": 2051.772922800228, + "p50": 2031.0940880153794, + "p90": 2680.799270607531, + "p95": 2715.9889496513642, + "p99": 2745.1213573251152 + }, + "ttft_ms": { + "mean": 148.5712587586022, + "p50": 137.2895505046472, + "p90": 142.76428538141772, + "p95": 147.17790617141873, + "p99": 403.30891362857074 + }, + "tpot_ms": { + "mean": 9.655633524071195, + "p50": 9.704072993671705, + "p90": 10.211875773304342, + "p95": 10.47933470059539, + "p99": 10.984298463796051 + }, + "itl_ms": { + "mean": 9.60982244299022, + "p50": 9.667875145867418, + "p90": 10.210687953745627, + "p95": 10.38416295380234, + "p99": 10.939679769297733 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_256.jsonl" + }, + { + "name": "c8_i2048_o4096", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.121544424968306, + "request_throughput": 3.0119606222291315, + "input_token_throughput": 6168.495354325261, + "output_token_throughput": 661.0500575637386, + "total_token_throughput": 6829.545411889, + "total_input_tokens": 327680, + "total_output_tokens": 35116, + "e2e_ms": { + "mean": 2250.238640499447, + "p50": 2060.5107424780726, + "p90": 3155.8005181024782, + "p95": 3788.7343745154812, + "p99": 4649.511583077255 + }, + "ttft_ms": { + "mean": 150.99034105223836, + "p50": 137.72946450626478, + "p90": 144.35978347901255, + "p95": 219.6447414258729, + "p99": 421.1795450851787 + }, + "tpot_ms": { + "mean": 9.576291877122781, + "p50": 9.656859571750678, + "p90": 10.295353155562944, + "p95": 10.442840379462343, + "p99": 10.576213952237916 + }, + "itl_ms": { + "mean": 9.521656809451594, + "p50": 9.60966083004923, + "p90": 10.244706533607456, + "p95": 10.41720936334391, + "p99": 10.524654688477527 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_4096.jsonl" + }, + { + "name": "c8_i2048_o512", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 50.97147612500703, + "request_throughput": 3.139010524388221, + "input_token_throughput": 6428.693553947077, + "output_token_throughput": 669.0408556417945, + "total_token_throughput": 7097.734409588871, + "total_input_tokens": 327680, + "total_output_tokens": 34102, + "e2e_ms": { + "mean": 2190.124648256824, + "p50": 2051.997427508468, + "p90": 3092.7031756960787, + "p95": 3392.3705204302682, + "p99": 4269.132630236562 + }, + "ttft_ms": { + "mean": 148.41127000145207, + "p50": 137.25388501188718, + "p90": 143.0399636097718, + "p95": 147.71226262673738, + "p99": 403.41484271222726 + }, + "tpot_ms": { + "mean": 9.604414580719462, + "p50": 9.644402433341643, + "p90": 10.206372463942852, + "p95": 10.483934730782448, + "p99": 11.082257841628142 + }, + "itl_ms": { + "mean": 9.547872710361627, + "p50": 9.566959381900393, + "p90": 10.16867574821862, + "p95": 10.445367728095432, + "p99": 10.99869270011619 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_2048_512.jsonl" + }, + { + "name": "c8_i262144_o128", + "config": { + "concurrency": 8, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 1333.5599152920186, + "request_throughput": 0.11997961108853795, + "input_token_throughput": 31451.935169193694, + "output_token_throughput": 12.611356870543945, + "total_token_throughput": 31464.546526064238, + "total_input_tokens": 41943040, + "total_output_tokens": 16818, + "e2e_ms": { + "mean": 59143.646888742296, + "p50": 59151.90828900086, + "p90": 59696.19761038339, + "p95": 61777.268645277945, + "p99": 74358.74609683407 + }, + "ttft_ms": { + "mean": 36128.44722178088, + "p50": 31553.45409095753, + "p90": 57830.65334893763, + "p95": 57997.25315597025, + "p99": 58592.27338458295 + }, + "tpot_ms": { + "mean": 215.94794021037245, + "p50": 256.24419141826854, + "p90": 391.74129292216816, + "p95": 392.1382184028426, + "p99": 446.9436230443744 + }, + "itl_ms": { + "mean": 215.80289381987473, + "p50": 255.43096417095512, + "p90": 391.74069926375523, + "p95": 392.1376104767094, + "p99": 446.94327607237824 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_262144_128.jsonl" + }, + { + "name": "c8_i4096_o1024", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 60.1764503219747, + "request_throughput": 2.6588474252621817, + "input_token_throughput": 10890.639053873896, + "output_token_throughput": 578.3824039765639, + "total_token_throughput": 11469.02145785046, + "total_input_tokens": 655360, + "total_output_tokens": 34805, + "e2e_ms": { + "mean": 2550.258225586367, + "p50": 2352.0323729899246, + "p90": 3329.045555065386, + "p95": 4642.49045046745, + "p99": 7535.8905110671185 + }, + "ttft_ms": { + "mean": 253.10391337516194, + "p50": 232.87756598438136, + "p90": 238.5712873074226, + "p95": 364.2884655972013, + "p99": 731.9332470308291 + }, + "tpot_ms": { + "mean": 10.57177228837578, + "p50": 10.64941300418702, + "p90": 11.587295458922837, + "p95": 11.887076822407384, + "p99": 12.167304394979784 + }, + "itl_ms": { + "mean": 10.507668801966785, + "p50": 10.586497942778886, + "p90": 11.535050429966855, + "p95": 11.721179619833572, + "p99": 11.979766313497109 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_1024.jsonl" + }, + { + "name": "c8_i4096_o128", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 40.354094801004976, + "request_throughput": 3.9649012272235473, + "input_token_throughput": 16240.23542670765, + "output_token_throughput": 506.0205191244052, + "total_token_throughput": 16746.255945832054, + "total_input_tokens": 655360, + "total_output_tokens": 20420, + "e2e_ms": { + "mean": 1721.3472404528147, + "p50": 1727.0378270186484, + "p90": 1749.9196672462858, + "p95": 1910.0958328694105, + "p99": 2181.081341035315 + }, + "ttft_ms": { + "mean": 376.50707827706356, + "p50": 259.70822700764984, + "p90": 686.7596228956245, + "p95": 695.3964474290842, + "p99": 727.2619360673707 + }, + "tpot_ms": { + "mean": 10.61910417605264, + "p50": 10.8540513521434, + "p90": 11.925979451263528, + "p95": 11.98689676731858, + "p99": 13.733626962350641 + }, + "itl_ms": { + "mean": 10.616558701551055, + "p50": 10.81030888415592, + "p90": 11.925486865403922, + "p95": 11.986494040658412, + "p99": 13.733118322708156 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_128.jsonl" + }, + { + "name": "c8_i4096_o2048", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.13547281298088, + "request_throughput": 2.800361878928021, + "input_token_throughput": 11470.282256089175, + "output_token_throughput": 593.2216595274147, + "total_token_throughput": 12063.50391561659, + "total_input_tokens": 655360, + "total_output_tokens": 33894, + "e2e_ms": { + "mean": 2478.8010334064893, + "p50": 2387.5871959899087, + "p90": 3337.4612528132275, + "p95": 3843.5530389077044, + "p99": 5407.4504248407875 + }, + "ttft_ms": { + "mean": 254.35324738755298, + "p50": 234.77283149259165, + "p90": 237.6235678908415, + "p95": 361.27230101847096, + "p99": 732.140513684717 + }, + "tpot_ms": { + "mean": 10.518332535585891, + "p50": 10.590422963040899, + "p90": 11.502042432841588, + "p95": 11.682779447372331, + "p99": 12.363350102408123 + }, + "itl_ms": { + "mean": 10.459223410056985, + "p50": 10.526562515166122, + "p90": 11.435158975709735, + "p95": 11.629216193575695, + "p99": 12.304858903853528 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_2048.jsonl" + }, + { + "name": "c8_i4096_o256", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.73408304201439, + "request_throughput": 2.923224271011949, + "input_token_throughput": 11973.526614064944, + "output_token_throughput": 575.2357260800701, + "total_token_throughput": 12548.762340145013, + "total_input_tokens": 655360, + "total_output_tokens": 31485, + "e2e_ms": { + "mean": 2342.5611934311746, + "p50": 2375.0031819799915, + "p90": 2998.0281604104675, + "p95": 3122.9925069375895, + "p99": 3247.983481837436 + }, + "ttft_ms": { + "mean": 252.75953688651498, + "p50": 232.09355198196135, + "p90": 238.3180244883988, + "p95": 366.29044984583743, + "p99": 732.2054903308162 + }, + "tpot_ms": { + "mean": 10.626070005954862, + "p50": 10.651171180348502, + "p90": 11.678796063279654, + "p95": 12.020385139763485, + "p99": 12.587150601270418 + }, + "itl_ms": { + "mean": 10.578348915412478, + "p50": 10.631525566033428, + "p90": 11.625460455067769, + "p95": 11.963682290983403, + "p99": 12.537233502371484 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_256.jsonl" + }, + { + "name": "c8_i4096_o4096", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 61.97299993701745, + "request_throughput": 2.5817694828813584, + "input_token_throughput": 10574.927801882044, + "output_token_throughput": 602.6011333637771, + "total_token_throughput": 11177.528935245822, + "total_input_tokens": 655360, + "total_output_tokens": 37345, + "e2e_ms": { + "mean": 2715.5006801924173, + "p50": 2296.5276565228123, + "p90": 3336.566875816788, + "p95": 3797.2634706238773, + "p99": 5487.115504498359 + }, + "ttft_ms": { + "mean": 251.6203917311941, + "p50": 233.67508800583892, + "p90": 237.69924038788304, + "p95": 276.3132474210587, + "p99": 733.8285815808922 + }, + "tpot_ms": { + "mean": 10.484129967546775, + "p50": 10.615274145465921, + "p90": 11.663851770502971, + "p95": 11.806859850762217, + "p99": 12.225997983203529 + }, + "itl_ms": { + "mean": 10.422449648022756, + "p50": 10.50003365653224, + "p90": 11.607559697366497, + "p95": 11.752852474078695, + "p99": 12.169660355465783 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_4096.jsonl" + }, + { + "name": "c8_i4096_o512", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.591448950988706, + "request_throughput": 2.7781902159843677, + "input_token_throughput": 11379.46712467197, + "output_token_throughput": 586.7537736158985, + "total_token_throughput": 11966.22089828787, + "total_input_tokens": 655360, + "total_output_tokens": 33792, + "e2e_ms": { + "mean": 2482.8999182365806, + "p50": 2328.508917009458, + "p90": 3322.9401516378853, + "p95": 4048.3197444671596, + "p99": 5636.745985752786 + }, + "ttft_ms": { + "mean": 255.020999507542, + "p50": 233.46155596664175, + "p90": 243.00072313053533, + "p95": 376.2320582260145, + "p99": 733.9472911530174 + }, + "tpot_ms": { + "mean": 10.53575250735599, + "p50": 10.566613968387138, + "p90": 11.549565030957893, + "p95": 11.703975537785546, + "p99": 12.172998480346749 + }, + "itl_ms": { + "mean": 10.477811694518088, + "p50": 10.5075818050335, + "p90": 11.473276370174169, + "p95": 11.617491250124376, + "p99": 11.994851243780436 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_4096_512.jsonl" + }, + { + "name": "c8_i512_o256", + "config": { + "concurrency": 8, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 48.34384520503227, + "request_throughput": 3.309625027165714, + "input_token_throughput": 1694.5280139088457, + "output_token_throughput": 686.9954150139231, + "total_token_throughput": 2381.5234289227687, + "total_input_tokens": 81920, + "total_output_tokens": 33212, + "e2e_ms": { + "mean": 2075.1491860271926, + "p50": 2173.637369007338, + "p90": 2598.430469626328, + "p95": 2669.6843849582365, + "p99": 2871.4014900499023 + }, + "ttft_ms": { + "mean": 125.67628707365657, + "p50": 116.35541348368861, + "p90": 131.15600130986414, + "p95": 194.69460808322768, + "p99": 258.2202711654827 + }, + "tpot_ms": { + "mean": 9.433366300506062, + "p50": 9.411735510561925, + "p90": 10.00254407070816, + "p95": 10.430004206536266, + "p99": 10.950972265487408 + }, + "itl_ms": { + "mean": 9.394523221295401, + "p50": 9.36648559752587, + "p90": 9.957444830914309, + "p95": 10.423651138739814, + "p99": 10.936330572939495 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_512_256.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 298.66033907805104, + "request_throughput": 0.5357256356632812, + "input_token_throughput": 35109.3152588288, + "output_token_throughput": 107.56366278551818, + "total_token_throughput": 35216.87892161432, + "total_input_tokens": 10485760, + "total_output_tokens": 32125, + "e2e_ms": { + "mean": 13001.392204627336, + "p50": 12542.045173991937, + "p90": 19688.615291757742, + "p95": 21336.68931651919, + "p99": 30533.505789504845 + }, + "ttft_ms": { + "mean": 3847.522275053052, + "p50": 3268.898812500993, + "p90": 5648.875438020332, + "p95": 5884.526047899271, + "p99": 9844.517542416572 + }, + "tpot_ms": { + "mean": 46.162656343687104, + "p50": 44.22957419128011, + "p90": 71.19218689531144, + "p95": 82.22329133258519, + "p99": 101.30540995954283 + }, + "itl_ms": { + "mean": 45.949579851959996, + "p50": 44.21014147421841, + "p90": 71.18646031193735, + "p95": 81.51694651542725, + "p99": 101.30496182853231 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 274.1067653300124, + "request_throughput": 0.5837141589970137, + "input_token_throughput": 38254.29112402829, + "output_token_throughput": 70.29742581070911, + "total_token_throughput": 38324.588549839, + "total_input_tokens": 10485760, + "total_output_tokens": 19269, + "e2e_ms": { + "mean": 12089.844919632378, + "p50": 12085.347602027468, + "p90": 12178.208685596474, + "p95": 12347.214980889112, + "p99": 12881.315556197076 + }, + "ttft_ms": { + "mean": 7223.655334466821, + "p50": 7265.156039007707, + "p90": 11034.40593361156, + "p95": 11101.746441455907, + "p99": 11262.27838820545 + }, + "tpot_ms": { + "mean": 40.500587106675695, + "p50": 40.765876922386276, + "p90": 72.42243729535613, + "p95": 72.53750583942507, + "p99": 72.63441232998551 + }, + "itl_ms": { + "mean": 40.4976505026951, + "p50": 40.765389234094, + "p90": 72.42207835310077, + "p95": 72.53694979015502, + "p99": 72.63400630344259 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_128.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 291.00532128399936, + "request_throughput": 0.549818124610347, + "input_token_throughput": 36032.8806144637, + "output_token_throughput": 98.87104425805563, + "total_token_throughput": 36131.751658721754, + "total_input_tokens": 10485760, + "total_output_tokens": 28772, + "e2e_ms": { + "mean": 12843.240155881358, + "p50": 12733.343144471291, + "p90": 17990.28853759519, + "p95": 18584.33374123124, + "p99": 21293.288793755928 + }, + "ttft_ms": { + "mean": 3804.886426846133, + "p50": 3264.9704460054636, + "p90": 5579.19040689012, + "p95": 5718.145171192011, + "p99": 9852.35885454632 + }, + "tpot_ms": { + "mean": 50.924454910755436, + "p50": 49.41619370337081, + "p90": 72.27905076555149, + "p95": 78.36170262279879, + "p99": 96.49280662046667 + }, + "itl_ms": { + "mean": 50.71989161569722, + "p50": 49.07127746615946, + "p90": 72.20096208478664, + "p95": 76.45671442829546, + "p99": 95.44811846482862 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_256.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 297.0369087840081, + "request_throughput": 0.5386535991604491, + "input_token_throughput": 35301.20227457919, + "output_token_throughput": 111.90191864058855, + "total_token_throughput": 35413.104193219784, + "total_input_tokens": 10485760, + "total_output_tokens": 33239, + "e2e_ms": { + "mean": 13139.025081034924, + "p50": 12636.850299983053, + "p90": 20860.50389088923, + "p95": 22983.13999388484, + "p99": 31590.46198525988 + }, + "ttft_ms": { + "mean": 3902.294257010726, + "p50": 3372.0412499969825, + "p90": 5779.39000299084, + "p95": 6055.86706744216, + "p99": 9835.604973481379 + }, + "tpot_ms": { + "mean": 44.23024008462845, + "p50": 44.65291751845124, + "p90": 66.10532247314792, + "p95": 73.93369533170166, + "p99": 83.81848196608775 + }, + "itl_ms": { + "mean": 44.01114226693558, + "p50": 44.573540121602136, + "p90": 65.53879755658635, + "p95": 73.88402448965356, + "p99": 83.77087468636375 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp4_custom_bench/results/20260711-133908/raw_outputs/vllm_0711_8_65536_512.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md deleted file mode 100644 index 288b634..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,126 +0,0 @@ -# dsv4_h200_vllm_tp4_custom_bench_no_fp8 - -## 目的 - -当 vLLM 使用 **TP=4**（Tensor Parallel = 4）部署时，单张 H200 上可以同时运行 **2 个独立服务**（8 卡 / 4 = 2 服务），每个服务独占 4 张 GPU： - -> **注意**：本实验变体 **不启用 `--kv-cache-dtype fp8`**，用于对比 FP8 KV Cache 与默认精度的性能差异。 - -| 服务 | 端口 | GPU 组 | -|------|------|--------| -| 1 | 30005 | 0, 1, 2, 3 | -| 2 | 30006 | 4, 5, 6, 7 | - -相比 TP=2 方案，TP=4 的每个服务拥有更大的 tensor-parallel 宽度，适合更高并发的单服务场景。本实验同样使用 **自定义压测客户端 `bench_client.py`**，通过负载均衡将请求均匀分发到 2 个服务上。 - -## 与 TP=2 方案的区别 - -| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | -|------|---------------|---------------| -| 服务数量 | 4 | 2 | -| 每服务 GPU | 2 | 4 | -| 单服务吞吐 | 较低 | 较高 | -| 集群总吞吐 | 相近 | 相近 | -| 适用场景 | 低并发、低延迟 | 高并发、大 batch | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端，支持多服务负载均衡（复用 TP=2 版本） | -| `config.env` | 实验配置（2 服务、端口 30005/30006、场景等） | -| `run_bench.sh` | 编排脚本：启动 2 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 同时启动 2 个 vLLM TP=4 服务（每服务 4 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动 2 个服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh - -# 跳过服务管理（已有 2 个服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp4_custom_bench/run_bench.sh - -# 单独运行压测客户端（多服务，带健康检查） -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --lb-strategy round_robin \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file /tmp/test.jsonl - -# 单独运行压测客户端（单服务，向后兼容） -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 8 --input-len 512 --output-len 256 --num-prompts 160 \ - --output-file /tmp/test_single.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp4_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp4_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 2 | 512 | 256 | 40 | 低并发，2 服务各 1 req | -| 4 | 512 | 256 | 80 | 中低并发 | -| 8 | 512 | 256 | 160 | 中等并发 | -| 16 | 512 | 256 | 320 | 中高并发 | -| 32 | 512 | 256 | 640 | 高并发 | -| 64 | 512 | 256 | 1280 | 很高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测集群吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 健康检查与故障自动跳过 - -同 TP=2 版本，详见原 README。 - -## 负载均衡策略 - -同 TP=2 版本，支持 `round_robin`（默认）、`random`、`least_conn`。 - -## 输出格式 - -同 TP=2 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 - -## 单独启动/停止 2 个服务 - -```bash -# 启动 2 个服务 -bash experiments/dsv4_h200_vllm_tp4_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -for port in 30005 30006; do - pid_file="experiments/dsv4_h200_vllm_tp4_custom_bench/server_port${port}.pid" - [[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true - rm -f "$pid_file" -done -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- 2 个服务共享同一模型文件，确保 `/data/models/DeepSeek-V4-Flash` 存在且可访问。 -- 每个服务占用约 4 张 GPU 的 90% 显存，确保无其他进程占用 GPU。 -- TP=4 相比 TP=2 单服务延迟更低（通信更少），但服务数量减半，低并发下可能无法充分利用集群。 -- 健康检查依赖 `/health` 端点，确保 vLLM 服务已启用该端点（vLLM 默认启用）。 -- 如果某个服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env deleted file mode 100644 index 8f784be..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,84 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005,30006" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For multi-service we include low-concurrency scenarios to show how -# requests spread across 2 services. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 0ec7941..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for multi-service vLLM TP=4 on 8x H200. -# -# This runs 2 independent vLLM services (TP=4 each) on GPU groups (0,1,2,3) and (4,5,6,7). -# bench_client.py distributes requests across all 2 services via round-robin LB, -# giving accurate cluster-wide throughput even at low per-service concurrency. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 2x vLLM TP=4 multi-service benchmark using bench_client.py (no FP8 KV cache)" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 4, - "num_services": 2, - "ports": "30005,30006", - "gpu_groups": "0,1,2,3|4,5,6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "2 512 256 40", "4 512 256 80", - "8 512 256 160", "16 512 256 320", "32 512 256 640", - "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (multi-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --ports "$PORTS" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --lb-strategy "$LB_STRATEGY" \ - --health-check-interval 5 \ - --health-max-failures 2 \ - --health-recovery-interval 10 \ - --request-max-retries 2 \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index 25cce7d..0000000 --- a/experiments/dsv4_h200_vllm_tp4_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# Start 2 independent vLLM TP=4 services on 8x H200. -# Each service runs on 4 GPUs: (0,1,2,3) and (4,5,6,7). -# This maximizes per-service throughput for higher-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=4 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 2 services, each on 4 GPUs. -# Port and GPU mapping. -SERVICES=( - "30005:0,1,2,3" - "30006:4,5,6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for all 2 services to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "All 2 services are healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Not all services became healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py index d8edc36..9deaa5b 100755 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py @@ -228,16 +228,22 @@ class LoadBalancer: # Token helpers # --------------------------------------------------------------------------- -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: +import uuid +def make_prompt(token_len: int) -> str: + prefix = str(uuid.uuid4()) + + words = [ + "the", "quick", "brown", "fox", + "jumps", "over", "lazy", "dog", + "benchmark", "performance", "latency" + ] + + tokens = [prefix] + + while len(tokens) < token_len: tokens.extend(words) - return " ".join(tokens[:needed]) + + return " ".join(tokens[:token_len]) # --------------------------------------------------------------------------- @@ -440,7 +446,36 @@ async def run_benchmark( return await send_request_with_retry( session, host, lb, model, idx, prompt, output_len, temperature ) + print("Starting warmup...") + warmup_per_service = 10 + warmup_requests = warmup_per_service * len(ports) + warmup_prompts = [ + make_prompt(input_len) + for _ in range(warmup_requests) + ] + + warmup_tasks = [ + send_request_with_retry( + session=session, + host=host, + lb=lb, + model=model, + request_id=-1, + prompt=p, + max_tokens=output_len, + temperature=temperature, + ) + for p in warmup_prompts + ] + + await asyncio.gather(*warmup_tasks) + + print("Warmup finished.") + + # 给 CUDA / Scheduler 一点时间稳定 + await asyncio.sleep(2) + tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] results = await asyncio.gather(*tasks) await health_checker.stop() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env index 0cc0836..d559ca6 100644 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/config.env @@ -32,51 +32,148 @@ LB_STRATEGY="round_robin" # Benchmark scenarios: "concurrency input_len output_len num_prompts". # For single-service TP=8 we cover standard concurrency gradients. +# concurrency input_len output_len num_prompts SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" + "1 512 256 20" + "2 512 256 40" + "4 512 256 80" + "8 512 256 160" + "16 512 256 320" + "32 512 256 640" + "64 512 256 1280" + "128 512 256 2560" + + "1 1024 128 20" + "1 1024 256 20" + "1 1024 512 20" + "1 1024 1024 20" + "1 1024 2048 20" + "1 1024 4096 20" + + "8 1024 128 160" + "8 1024 256 160" + "8 1024 512 160" + "8 1024 1024 160" + "8 1024 2048 160" + "8 1024 4096 160" + + "32 1024 128 640" + "32 1024 256 640" + "32 1024 512 640" + "32 1024 1024 640" + "32 1024 2048 640" + "32 1024 4096 640" + + "128 1024 128 2560" + "128 1024 256 2560" + "128 1024 512 2560" + "128 1024 1024 2560" + "128 1024 2048 2560" + "128 1024 4096 2560" + + "1 2048 128 20" + "1 2048 256 20" + "1 2048 512 20" + "1 2048 1024 20" + "1 2048 2048 20" + "1 2048 4096 20" + + "8 2048 128 160" + "8 2048 256 160" + "8 2048 512 160" + "8 2048 1024 160" + "8 2048 2048 160" + "8 2048 4096 160" + + "32 2048 128 640" + "32 2048 256 640" + "32 2048 512 640" + "32 2048 1024 640" + "32 2048 2048 640" + "32 2048 4096 640" + + "128 2048 128 2560" + "128 2048 256 2560" + "128 2048 512 2560" + "128 2048 1024 2560" + "128 2048 2048 2560" + "128 2048 4096 2560" + + "1 4096 128 20" + "1 4096 256 20" + "1 4096 512 20" + "1 4096 1024 20" + "1 4096 2048 20" + "1 4096 4096 20" + + "8 4096 128 160" + "8 4096 256 160" + "8 4096 512 160" + "8 4096 1024 160" + "8 4096 2048 160" + "8 4096 4096 160" + + "32 4096 128 640" + "32 4096 256 640" + "32 4096 512 640" + "32 4096 1024 640" + "32 4096 2048 640" + "32 4096 4096 640" + + "128 4096 128 2560" + "128 4096 256 2560" + "128 4096 512 2560" + "128 4096 1024 2560" + "128 4096 2048 2560" + "128 4096 4096 2560" + + "1 16384 128 20" + "1 16384 256 20" + "1 16384 512 20" + "1 16384 1024 20" + "1 16384 2048 20" + + "8 16384 128 160" + "8 16384 256 160" + "8 16384 512 160" + "8 16384 1024 160" + "8 16384 2048 160" + + "32 16384 128 640" + "32 16384 256 640" + "32 16384 512 640" + "32 16384 1024 640" + "32 16384 2048 640" + + "128 16384 128 2560" + "128 16384 256 2560" + "128 16384 512 2560" + "128 16384 1024 2560" + "128 16384 2048 2560" + + + "1 65536 128 20" + "1 65536 256 20" + "1 65536 512 20" + "1 65536 1024 20" + + "8 65536 128 160" + "8 65536 256 160" + "8 65536 512 160" + "8 65536 1024 160" + "32 65536 1024 160" + + "1 131072 128 20" + "1 131072 256 20" + "1 131072 512 20" + + "8 131072 512 160" + + "1 262144 256 20" + + "8 262144 128 160" + + "1 524288 128 20" ) # Server start script bundled with this experiment. diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md new file mode 100644 index 0000000..c8cb28f --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/report.md @@ -0,0 +1,129 @@ +# H200 vLLM TP=2 Custom Benchmark Report + +- **Client**: `bench_client.py` (async OpenAI API, per-request timing) +- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding) +- **Result root**: `/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756` + +## Results + +| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| c128_i1024_o1024 | 128 | 1024 | 1024 | 206.75 | 2560 | 0 | 12.38 | 12679.34 | 2588.49 | 15267.83 | 355.51 | 541.00 | 3399.25 | 46.35 | 50.58 | 52.54 | 9995.61 | 13713.98 | 15284.95 | ✅ | +| c128_i1024_o128 | 128 | 1024 | 128 | 133.01 | 2560 | 0 | 19.25 | 19708.67 | 2459.70 | 22168.36 | 1183.32 | 1479.29 | 3426.82 | 41.60 | 47.50 | 49.22 | 6458.71 | 7012.22 | 8759.52 | ✅ | +| c128_i1024_o2048 | 128 | 1024 | 2048 | 208.80 | 2560 | 0 | 12.26 | 12554.51 | 2547.16 | 15101.67 | 351.46 | 493.93 | 3429.21 | 47.17 | 52.36 | 59.29 | 10106.58 | 13980.86 | 15725.90 | ✅ | +| c128_i1024_o256 | 128 | 1024 | 256 | 205.11 | 2560 | 0 | 12.48 | 12780.63 | 2561.23 | 15341.86 | 351.22 | 456.94 | 3426.54 | 46.93 | 50.36 | 51.39 | 9935.31 | 12606.55 | 13028.92 | ✅ | +| c128_i1024_o4096 | 128 | 1024 | 4096 | 208.63 | 2560 | 0 | 12.27 | 12564.96 | 2562.18 | 15127.14 | 352.13 | 475.05 | 3401.06 | 46.82 | 50.35 | 51.59 | 10079.33 | 13938.76 | 15626.71 | ✅ | +| c128_i1024_o512 | 128 | 1024 | 512 | 210.21 | 2560 | 0 | 12.18 | 12470.38 | 2534.38 | 15004.76 | 355.66 | 519.46 | 3397.33 | 47.25 | 52.84 | 55.05 | 10138.54 | 13931.68 | 15398.92 | ✅ | +| c128_i16384_o128 | 128 | 16384 | 128 | 1359.07 | 2560 | 0 | 1.88 | 30861.66 | 240.91 | 31102.57 | 34308.65 | 34726.75 | 53850.14 | 257.93 | 261.13 | 261.25 | 67037.51 | 67882.99 | 86482.77 | ❌ | +| c128_i16384_o256 | 128 | 16384 | 256 | 1383.21 | 2560 | 0 | 1.85 | 30323.03 | 393.12 | 30716.14 | 13674.31 | 16619.00 | 53959.97 | 259.70 | 265.74 | 266.01 | 68522.28 | 82348.90 | 113143.30 | ❌ | +| c128_i16384_o512 | 128 | 16384 | 512 | 1394.58 | 2560 | 0 | 1.84 | 30075.73 | 449.14 | 30524.87 | 6929.19 | 14006.30 | 53929.18 | 256.42 | 266.39 | 266.68 | 68997.86 | 134484.18 | 142208.84 | ❌ | +| c128_i2048_o1024 | 128 | 2048 | 1024 | 274.18 | 2560 | 0 | 9.34 | 19121.84 | 2068.13 | 21189.97 | 541.55 | 764.67 | 6604.67 | 57.69 | 62.18 | 65.53 | 13253.98 | 21801.42 | 28048.56 | ⚠️ | +| c128_i2048_o128 | 128 | 2048 | 128 | 212.99 | 2560 | 0 | 12.02 | 24615.40 | 1530.02 | 26145.42 | 1293.09 | 1997.49 | 6703.31 | 72.33 | 80.38 | 81.78 | 10431.32 | 11370.95 | 15884.14 | ⚠️ | +| c128_i2048_o2048 | 128 | 2048 | 2048 | 272.06 | 2560 | 0 | 9.41 | 19270.69 | 2055.16 | 21325.85 | 542.66 | 696.46 | 6595.59 | 58.02 | 62.67 | 65.23 | 13143.51 | 21512.40 | 28583.36 | ⚠️ | +| c128_i2048_o256 | 128 | 2048 | 256 | 263.83 | 2560 | 0 | 9.70 | 19872.28 | 1986.56 | 21858.84 | 552.74 | 766.94 | 6667.42 | 60.36 | 64.80 | 67.61 | 12841.30 | 16455.00 | 19770.97 | ⚠️ | +| c128_i2048_o4096 | 128 | 2048 | 4096 | 274.18 | 2560 | 0 | 9.34 | 19121.82 | 2075.59 | 21197.41 | 538.85 | 772.70 | 6666.24 | 57.59 | 61.73 | 65.98 | 13271.96 | 22132.76 | 28679.85 | ⚠️ | +| c128_i2048_o512 | 128 | 2048 | 512 | 271.80 | 2560 | 0 | 9.42 | 19289.73 | 2074.26 | 21363.99 | 544.24 | 780.74 | 6725.67 | 57.75 | 61.62 | 65.40 | 13185.12 | 21822.20 | 28335.44 | ⚠️ | +| c128_i4096_o1024 | 128 | 4096 | 1024 | 424.83 | 2560 | 0 | 6.03 | 24682.53 | 1273.81 | 25956.34 | 1014.26 | 1488.91 | 13325.90 | 93.93 | 103.74 | 115.24 | 20697.67 | 35507.95 | 47170.14 | ⚠️ | +| c128_i4096_o128 | 128 | 4096 | 128 | 371.42 | 2560 | 0 | 6.89 | 28231.71 | 873.71 | 29105.42 | 1687.74 | 2056.57 | 13344.42 | 132.08 | 142.11 | 145.58 | 18310.58 | 19462.37 | 30169.28 | ⚠️ | +| c128_i4096_o2048 | 128 | 4096 | 2048 | 425.25 | 2560 | 0 | 6.02 | 24657.77 | 1288.03 | 25945.80 | 1019.46 | 1464.89 | 13332.86 | 93.13 | 101.52 | 111.69 | 20779.35 | 36358.76 | 50447.50 | ⚠️ | +| c128_i4096_o256 | 128 | 4096 | 256 | 413.80 | 2560 | 0 | 6.19 | 25340.14 | 1217.79 | 26557.93 | 1065.75 | 1554.31 | 13333.89 | 98.37 | 108.53 | 114.97 | 20311.65 | 26976.60 | 33054.57 | ⚠️ | +| c128_i4096_o4096 | 128 | 4096 | 4096 | 423.76 | 2560 | 0 | 6.04 | 24744.50 | 1282.56 | 26027.06 | 1040.84 | 1941.31 | 13328.94 | 93.34 | 101.81 | 111.13 | 20723.12 | 35255.54 | 48374.31 | ⚠️ | +| c128_i4096_o512 | 128 | 4096 | 512 | 423.70 | 2560 | 0 | 6.04 | 24748.17 | 1274.42 | 26022.58 | 1032.52 | 1562.56 | 13323.34 | 93.86 | 103.68 | 112.51 | 20669.59 | 35396.55 | 49034.77 | ⚠️ | +| c128_i512_o256 | 128 | 512 | 256 | 198.05 | 2560 | 0 | 12.93 | 6618.16 | 2673.83 | 9291.99 | 290.66 | 501.00 | 1944.87 | 45.16 | 52.68 | 57.19 | 9585.92 | 13135.34 | 14237.26 | ✅ | +| c16_i512_o256 | 16 | 512 | 256 | 70.00 | 320 | 0 | 4.57 | 2340.54 | 954.25 | 3294.79 | 155.99 | 299.17 | 421.01 | 14.77 | 16.43 | 17.44 | 3227.91 | 4110.18 | 4253.24 | ✅ | +| c1_i1024_o1024 | 1 | 1024 | 1024 | 41.00 | 20 | 0 | 0.49 | 499.49 | 102.02 | 601.51 | 113.27 | 116.29 | 137.79 | 6.73 | 6.74 | 6.74 | 1513.65 | 1949.27 | 1998.66 | ✅ | +| c1_i1024_o128 | 1 | 1024 | 128 | 26.62 | 20 | 0 | 0.75 | 769.29 | 96.16 | 865.45 | 112.10 | 114.09 | 127.55 | 6.69 | 6.70 | 6.70 | 961.87 | 964.11 | 977.09 | ✅ | +| c1_i1024_o2048 | 1 | 1024 | 2048 | 40.37 | 20 | 0 | 0.50 | 507.26 | 100.39 | 607.65 | 113.54 | 118.17 | 129.38 | 6.73 | 6.75 | 6.75 | 1470.10 | 1949.87 | 2093.40 | ✅ | +| c1_i1024_o256 | 1 | 1024 | 256 | 39.66 | 20 | 0 | 0.50 | 516.37 | 101.41 | 617.77 | 112.03 | 113.19 | 128.94 | 6.73 | 6.74 | 6.74 | 1457.72 | 1819.60 | 1819.96 | ✅ | +| c1_i1024_o4096 | 1 | 1024 | 4096 | 39.07 | 20 | 0 | 0.51 | 524.19 | 100.38 | 624.57 | 126.46 | 178.17 | 178.35 | 6.73 | 6.74 | 6.75 | 1439.54 | 1773.78 | 1831.83 | ✅ | +| c1_i1024_o512 | 1 | 1024 | 512 | 38.86 | 20 | 0 | 0.51 | 527.06 | 98.62 | 625.68 | 113.09 | 114.42 | 136.00 | 6.73 | 6.74 | 6.75 | 1396.05 | 1797.45 | 1820.39 | ✅ | +| c1_i16384_o1024 | 1 | 16384 | 1024 | 60.02 | 20 | 0 | 0.33 | 5459.87 | 74.28 | 5534.15 | 621.87 | 633.10 | 637.21 | 6.82 | 6.83 | 6.85 | 2136.32 | 2962.66 | 3211.92 | ✅ | +| c1_i16384_o128 | 1 | 16384 | 128 | 41.67 | 20 | 0 | 0.48 | 7862.85 | 60.88 | 7923.72 | 628.49 | 631.58 | 636.36 | 6.77 | 6.81 | 6.81 | 1480.09 | 1482.52 | 1482.98 | ✅ | +| c1_i16384_o2048 | 1 | 16384 | 2048 | 63.80 | 20 | 0 | 0.31 | 5136.27 | 80.76 | 5217.03 | 630.15 | 636.36 | 636.98 | 6.82 | 6.83 | 6.85 | 2381.45 | 4577.47 | 5209.66 | ✅ | +| c1_i16384_o256 | 1 | 16384 | 256 | 56.44 | 20 | 0 | 0.35 | 5805.62 | 73.65 | 5879.27 | 630.48 | 638.61 | 639.04 | 6.82 | 6.85 | 6.85 | 2040.24 | 2358.86 | 2362.47 | ✅ | +| c1_i16384_o512 | 1 | 16384 | 512 | 70.93 | 20 | 0 | 0.28 | 4619.51 | 85.87 | 4705.38 | 621.45 | 629.96 | 645.42 | 6.82 | 6.85 | 6.85 | 2692.18 | 4094.39 | 4094.98 | ✅ | +| c1_i2048_o1024 | 1 | 2048 | 1024 | 43.34 | 20 | 0 | 0.46 | 945.05 | 102.00 | 1047.06 | 123.15 | 127.95 | 148.69 | 6.80 | 6.81 | 6.81 | 1619.33 | 2410.40 | 2903.39 | ✅ | +| c1_i2048_o128 | 1 | 2048 | 128 | 27.02 | 20 | 0 | 0.74 | 1515.96 | 94.12 | 1610.08 | 122.85 | 126.65 | 147.47 | 6.77 | 6.77 | 6.82 | 976.42 | 983.68 | 1006.22 | ✅ | +| c1_i2048_o2048 | 1 | 2048 | 2048 | 43.35 | 20 | 0 | 0.46 | 944.86 | 101.45 | 1046.31 | 123.06 | 128.19 | 147.54 | 6.80 | 6.83 | 6.84 | 1611.88 | 2108.88 | 2496.01 | ✅ | +| c1_i2048_o256 | 1 | 2048 | 256 | 37.22 | 20 | 0 | 0.54 | 1100.59 | 96.68 | 1197.27 | 122.80 | 125.69 | 145.02 | 6.81 | 6.82 | 6.83 | 1340.69 | 1621.05 | 1775.55 | ✅ | +| c1_i2048_o4096 | 1 | 2048 | 4096 | 40.11 | 20 | 0 | 0.50 | 1021.08 | 101.61 | 1122.69 | 123.10 | 127.51 | 146.72 | 6.80 | 6.82 | 6.82 | 1502.66 | 2076.61 | 2137.65 | ✅ | +| c1_i2048_o512 | 1 | 2048 | 512 | 37.02 | 20 | 0 | 0.54 | 1106.44 | 96.52 | 1202.96 | 122.80 | 127.33 | 146.10 | 6.81 | 6.82 | 6.82 | 1331.89 | 1558.94 | 1578.90 | ✅ | +| c1_i4096_o1024 | 1 | 4096 | 1024 | 43.12 | 20 | 0 | 0.46 | 1899.75 | 99.83 | 1999.59 | 186.35 | 190.63 | 190.95 | 6.80 | 6.81 | 6.81 | 1644.18 | 2109.89 | 2311.93 | ✅ | +| c1_i4096_o128 | 1 | 4096 | 128 | 28.91 | 20 | 0 | 0.69 | 2833.86 | 88.56 | 2922.42 | 185.86 | 190.38 | 191.25 | 6.75 | 6.76 | 6.76 | 1043.43 | 1045.36 | 1047.58 | ✅ | +| c1_i4096_o2048 | 1 | 4096 | 2048 | 42.94 | 20 | 0 | 0.47 | 1907.88 | 96.51 | 2004.39 | 186.77 | 193.30 | 193.74 | 6.80 | 6.81 | 6.81 | 1589.25 | 2097.82 | 2452.99 | ✅ | +| c1_i4096_o256 | 1 | 4096 | 256 | 40.43 | 20 | 0 | 0.49 | 2026.29 | 93.65 | 2119.94 | 189.05 | 207.59 | 207.83 | 6.80 | 6.81 | 6.82 | 1469.51 | 1913.76 | 1913.87 | ✅ | +| c1_i4096_o4096 | 1 | 4096 | 4096 | 43.12 | 20 | 0 | 0.46 | 1899.82 | 96.29 | 1996.11 | 187.93 | 205.29 | 207.29 | 6.80 | 6.82 | 6.82 | 1593.63 | 2383.61 | 2729.94 | ✅ | +| c1_i4096_o512 | 1 | 4096 | 512 | 41.37 | 20 | 0 | 0.48 | 1980.16 | 89.19 | 2069.35 | 186.00 | 187.35 | 190.66 | 6.81 | 6.81 | 6.82 | 1434.80 | 2102.19 | 2396.89 | ✅ | +| c1_i512_o256 | 1 | 512 | 256 | 39.88 | 20 | 0 | 0.50 | 256.78 | 96.80 | 353.58 | 118.18 | 176.60 | 184.48 | 6.72 | 6.75 | 6.75 | 1408.44 | 1822.85 | 1871.30 | ✅ | +| c2_i512_o256 | 2 | 512 | 256 | 41.58 | 40 | 0 | 0.96 | 492.58 | 196.05 | 688.62 | 173.55 | 211.08 | 357.86 | 7.61 | 8.16 | 8.40 | 1706.70 | 2108.24 | 2165.31 | ✅ | +| c32_i1024_o1024 | 32 | 1024 | 1024 | 101.71 | 640 | 0 | 6.29 | 6443.13 | 1317.59 | 7760.73 | 197.93 | 311.27 | 984.94 | 22.03 | 23.78 | 24.21 | 4796.09 | 6445.23 | 7146.16 | ✅ | +| c32_i1024_o128 | 32 | 1024 | 128 | 53.75 | 640 | 0 | 11.91 | 12192.47 | 1523.87 | 13716.35 | 681.10 | 1046.63 | 1091.53 | 14.34 | 16.98 | 18.60 | 2502.18 | 2579.44 | 2641.32 | ✅ | +| c32_i1024_o2048 | 32 | 1024 | 2048 | 101.76 | 640 | 0 | 6.29 | 6440.44 | 1316.74 | 7757.18 | 204.19 | 335.85 | 1145.74 | 21.89 | 24.02 | 24.53 | 4767.74 | 6630.19 | 7109.51 | ✅ | +| c32_i1024_o256 | 32 | 1024 | 256 | 99.60 | 640 | 0 | 6.43 | 6579.98 | 1309.14 | 7889.12 | 200.93 | 331.50 | 921.88 | 22.15 | 24.50 | 25.32 | 4694.16 | 6056.40 | 6274.46 | ✅ | +| c32_i1024_o4096 | 32 | 1024 | 4096 | 103.10 | 640 | 0 | 6.21 | 6356.26 | 1309.63 | 7665.88 | 205.97 | 316.61 | 1157.52 | 22.04 | 24.05 | 24.79 | 4833.69 | 6458.13 | 7314.04 | ✅ | +| c32_i1024_o512 | 32 | 1024 | 512 | 103.06 | 640 | 0 | 6.21 | 6359.18 | 1304.32 | 7663.50 | 196.61 | 340.27 | 945.16 | 22.38 | 25.38 | 27.97 | 4872.21 | 6788.10 | 7689.83 | ✅ | +| c32_i16384_o1024 | 32 | 16384 | 1024 | 398.20 | 640 | 0 | 1.61 | 26333.15 | 402.85 | 26735.99 | 1772.26 | 3387.53 | 13518.46 | 70.46 | 85.49 | 101.84 | 19184.30 | 36508.49 | 54198.78 | ❌ | +| c32_i16384_o128 | 32 | 16384 | 128 | 359.20 | 640 | 0 | 1.78 | 29191.93 | 227.55 | 29419.48 | 3438.60 | 3983.92 | 13516.26 | 111.13 | 127.95 | 130.66 | 17518.50 | 19731.45 | 27923.40 | ❌ | +| c32_i16384_o2048 | 32 | 16384 | 2048 | 398.63 | 640 | 0 | 1.61 | 26304.81 | 403.52 | 26708.32 | 1715.04 | 2963.25 | 13514.42 | 70.47 | 86.65 | 95.98 | 19175.98 | 37536.35 | 50504.81 | ⚠️ | +| c32_i16384_o256 | 32 | 16384 | 256 | 383.92 | 640 | 0 | 1.67 | 27312.05 | 348.16 | 27660.21 | 1826.81 | 3388.69 | 13517.47 | 80.79 | 92.93 | 101.63 | 18646.62 | 24405.54 | 29695.09 | ❌ | +| c32_i16384_o512 | 32 | 16384 | 512 | 393.37 | 640 | 0 | 1.63 | 26656.12 | 383.78 | 27039.90 | 1747.04 | 3400.82 | 13513.48 | 73.78 | 90.30 | 96.96 | 19000.89 | 36338.55 | 41160.95 | ❌ | +| c32_i2048_o1024 | 32 | 2048 | 1024 | 112.12 | 640 | 0 | 5.71 | 11690.22 | 1270.45 | 12960.67 | 246.53 | 373.10 | 1896.26 | 22.21 | 24.25 | 25.25 | 5165.11 | 8407.83 | 11262.12 | ✅ | +| c32_i2048_o128 | 32 | 2048 | 128 | 73.91 | 640 | 0 | 8.66 | 17733.36 | 1105.41 | 18838.77 | 904.68 | 1464.61 | 1844.76 | 20.43 | 26.38 | 27.06 | 3492.80 | 3746.52 | 4348.11 | ✅ | +| c32_i2048_o2048 | 32 | 2048 | 2048 | 111.79 | 640 | 0 | 5.73 | 11725.08 | 1282.42 | 13007.50 | 246.50 | 442.18 | 1847.04 | 22.17 | 24.28 | 25.62 | 5185.73 | 8810.37 | 10858.07 | ✅ | +| c32_i2048_o256 | 32 | 2048 | 256 | 103.61 | 640 | 0 | 6.18 | 12650.11 | 1258.11 | 13908.22 | 240.98 | 367.88 | 1848.70 | 22.80 | 24.74 | 25.47 | 4859.84 | 6298.45 | 6687.99 | ✅ | +| c32_i2048_o4096 | 32 | 2048 | 4096 | 106.84 | 640 | 0 | 5.99 | 12268.59 | 1282.15 | 13550.74 | 250.26 | 439.47 | 1837.79 | 22.45 | 24.65 | 25.84 | 5036.07 | 8060.13 | 10159.85 | ✅ | +| c32_i2048_o512 | 32 | 2048 | 512 | 111.06 | 640 | 0 | 5.76 | 11802.41 | 1281.65 | 13084.06 | 242.26 | 437.24 | 1893.17 | 22.15 | 24.51 | 25.55 | 5136.64 | 8816.12 | 11310.11 | ✅ | +| c32_i4096_o1024 | 32 | 4096 | 1024 | 141.80 | 640 | 0 | 4.51 | 18486.39 | 947.32 | 19433.71 | 432.85 | 813.49 | 3483.06 | 30.04 | 33.92 | 36.32 | 6702.72 | 11605.60 | 14590.75 | ✅ | +| c32_i4096_o128 | 32 | 4096 | 128 | 114.34 | 640 | 0 | 5.60 | 22925.90 | 710.98 | 23636.88 | 1053.81 | 1909.50 | 3485.20 | 35.10 | 41.98 | 43.26 | 5478.89 | 6106.50 | 7458.03 | ✅ | +| c32_i4096_o2048 | 32 | 4096 | 2048 | 144.45 | 640 | 0 | 4.43 | 18147.38 | 939.67 | 19087.05 | 434.39 | 747.31 | 3483.20 | 29.88 | 33.53 | 35.93 | 6700.04 | 10714.15 | 14733.26 | ✅ | +| c32_i4096_o256 | 32 | 4096 | 256 | 137.28 | 640 | 0 | 4.66 | 19095.92 | 921.51 | 20017.42 | 441.00 | 731.75 | 3478.68 | 30.98 | 34.42 | 36.58 | 6528.50 | 8638.60 | 10034.50 | ✅ | +| c32_i4096_o4096 | 32 | 4096 | 4096 | 143.01 | 640 | 0 | 4.48 | 18330.54 | 961.50 | 19292.04 | 429.61 | 749.16 | 3469.34 | 29.60 | 33.17 | 35.01 | 6743.49 | 10500.86 | 14281.37 | ✅ | +| c32_i4096_o512 | 32 | 4096 | 512 | 143.19 | 640 | 0 | 4.47 | 18307.18 | 929.29 | 19236.47 | 430.74 | 691.68 | 3532.95 | 30.57 | 34.15 | 36.96 | 6735.35 | 10401.62 | 14892.06 | ✅ | +| c32_i512_o256 | 32 | 512 | 256 | 98.73 | 640 | 0 | 6.48 | 3319.01 | 1330.82 | 4649.83 | 186.98 | 378.34 | 678.28 | 21.82 | 23.82 | 24.54 | 4649.84 | 6039.51 | 6163.96 | ✅ | +| c4_i512_o256 | 4 | 512 | 256 | 46.00 | 80 | 0 | 1.74 | 890.42 | 360.47 | 1250.89 | 164.64 | 347.47 | 374.55 | 8.92 | 10.51 | 11.09 | 1997.44 | 2523.64 | 2675.43 | ✅ | +| c64_i512_o256 | 64 | 512 | 256 | 140.30 | 1280 | 0 | 9.12 | 4671.07 | 1901.53 | 6572.60 | 229.06 | 453.78 | 1147.81 | 31.26 | 35.35 | 36.92 | 6712.71 | 8741.37 | 9237.35 | ✅ | +| c8_i1024_o1024 | 8 | 1024 | 1024 | 54.01 | 160 | 0 | 2.96 | 3033.30 | 621.77 | 3655.07 | 144.65 | 218.00 | 394.00 | 10.80 | 11.82 | 12.24 | 2395.48 | 3214.61 | 3506.80 | ✅ | +| c8_i1024_o128 | 8 | 1024 | 128 | 30.89 | 160 | 0 | 5.18 | 5304.66 | 662.82 | 5967.48 | 307.64 | 345.56 | 389.57 | 8.30 | 9.77 | 9.80 | 1360.75 | 1376.96 | 1412.43 | ✅ | +| c8_i1024_o2048 | 8 | 1024 | 2048 | 53.44 | 160 | 0 | 2.99 | 3065.61 | 627.08 | 3692.69 | 147.14 | 284.31 | 375.40 | 10.76 | 11.60 | 12.31 | 2391.87 | 3190.58 | 3558.55 | ✅ | +| c8_i1024_o256 | 8 | 1024 | 256 | 52.32 | 160 | 0 | 3.06 | 3131.63 | 620.55 | 3752.18 | 144.06 | 216.85 | 383.28 | 10.89 | 11.75 | 12.29 | 2345.43 | 2965.11 | 3042.16 | ✅ | +| c8_i1024_o4096 | 8 | 1024 | 4096 | 55.32 | 160 | 0 | 2.89 | 2961.58 | 621.49 | 3583.07 | 144.43 | 220.03 | 380.43 | 10.73 | 11.56 | 12.09 | 2438.42 | 3382.06 | 3756.38 | ✅ | +| c8_i1024_o512 | 8 | 1024 | 512 | 54.03 | 160 | 0 | 2.96 | 3032.14 | 632.21 | 3664.35 | 144.40 | 216.10 | 375.86 | 10.74 | 11.48 | 11.76 | 2425.82 | 3208.35 | 3616.77 | ✅ | +| c8_i16384_o1024 | 8 | 16384 | 1024 | 138.97 | 160 | 0 | 1.15 | 18863.47 | 287.59 | 19151.06 | 967.72 | 1600.40 | 3663.33 | 21.47 | 27.32 | 29.59 | 6298.76 | 12516.84 | 14588.60 | ✅ | +| c8_i16384_o128 | 8 | 16384 | 128 | 111.93 | 160 | 0 | 1.43 | 23420.44 | 181.33 | 23601.76 | 2556.64 | 4119.31 | 4136.10 | 20.74 | 33.97 | 34.00 | 5167.54 | 5417.75 | 5933.67 | ⚠️ | +| c8_i16384_o2048 | 8 | 16384 | 2048 | 136.26 | 160 | 0 | 1.17 | 19238.95 | 275.01 | 19513.96 | 961.94 | 1572.87 | 3673.56 | 22.52 | 28.43 | 30.68 | 6163.97 | 11684.40 | 14272.99 | ✅ | +| c8_i16384_o256 | 8 | 16384 | 256 | 127.82 | 160 | 0 | 1.25 | 20509.06 | 253.69 | 20762.74 | 1027.09 | 1938.52 | 3668.72 | 24.19 | 29.35 | 30.47 | 5890.64 | 7584.77 | 7970.70 | ✅ | +| c8_i16384_o512 | 8 | 16384 | 512 | 136.61 | 160 | 0 | 1.17 | 19188.83 | 286.09 | 19474.92 | 969.45 | 1904.35 | 3667.98 | 21.65 | 26.89 | 29.25 | 6250.80 | 10966.73 | 12572.72 | ✅ | +| c8_i2048_o1024 | 8 | 2048 | 1024 | 59.22 | 160 | 0 | 2.70 | 5533.53 | 606.82 | 6140.34 | 155.89 | 244.40 | 621.82 | 10.83 | 11.86 | 12.11 | 2572.18 | 4486.64 | 5823.30 | ✅ | +| c8_i2048_o128 | 8 | 2048 | 128 | 36.79 | 160 | 0 | 4.35 | 8907.76 | 554.37 | 9462.13 | 337.85 | 507.02 | 613.47 | 10.27 | 12.09 | 12.51 | 1636.65 | 1709.00 | 1726.45 | ✅ | +| c8_i2048_o2048 | 8 | 2048 | 2048 | 57.55 | 160 | 0 | 2.78 | 5693.86 | 626.97 | 6320.83 | 156.06 | 275.62 | 603.04 | 10.76 | 11.60 | 11.99 | 2571.87 | 4101.50 | 5201.48 | ✅ | +| c8_i2048_o256 | 8 | 2048 | 256 | 52.19 | 160 | 0 | 3.07 | 6279.15 | 604.00 | 6883.15 | 158.73 | 268.03 | 611.54 | 11.13 | 12.15 | 12.45 | 2334.26 | 3004.26 | 3124.59 | ✅ | +| c8_i2048_o4096 | 8 | 2048 | 4096 | 59.22 | 160 | 0 | 2.70 | 5533.71 | 614.83 | 6148.54 | 164.46 | 304.53 | 685.46 | 10.84 | 11.77 | 11.98 | 2624.68 | 4094.22 | 5661.19 | ✅ | +| c8_i2048_o512 | 8 | 2048 | 512 | 57.76 | 160 | 0 | 2.77 | 5672.85 | 605.80 | 6278.66 | 154.18 | 248.11 | 617.88 | 10.86 | 11.81 | 12.00 | 2521.32 | 3808.31 | 4671.03 | ✅ | +| c8_i4096_o1024 | 8 | 4096 | 1024 | 64.92 | 160 | 0 | 2.46 | 10094.27 | 518.13 | 10612.40 | 256.33 | 442.48 | 1016.15 | 12.36 | 13.77 | 14.46 | 2846.88 | 4602.44 | 5170.71 | ✅ | +| c8_i4096_o128 | 8 | 4096 | 128 | 46.47 | 160 | 0 | 3.44 | 14102.60 | 440.04 | 14542.64 | 707.03 | 1047.28 | 1071.50 | 10.92 | 15.03 | 15.06 | 2091.88 | 2145.56 | 2225.52 | ✅ | +| c8_i4096_o2048 | 8 | 4096 | 2048 | 66.29 | 160 | 0 | 2.41 | 9886.22 | 513.33 | 10399.56 | 253.25 | 423.32 | 1022.59 | 12.39 | 14.05 | 14.66 | 2865.36 | 4853.71 | 6028.24 | ✅ | +| c8_i4096_o256 | 8 | 4096 | 256 | 62.26 | 160 | 0 | 2.57 | 10526.27 | 516.50 | 11042.77 | 256.86 | 457.13 | 1013.55 | 12.63 | 14.05 | 14.57 | 2789.27 | 3625.36 | 3833.62 | ✅ | +| c8_i4096_o4096 | 8 | 4096 | 4096 | 64.28 | 160 | 0 | 2.49 | 10195.14 | 543.84 | 10738.99 | 248.19 | 462.57 | 1014.75 | 12.27 | 13.68 | 14.32 | 2921.13 | 5570.16 | 8112.52 | ✅ | +| c8_i4096_o512 | 8 | 4096 | 512 | 65.25 | 160 | 0 | 2.45 | 10043.48 | 538.74 | 10582.22 | 247.69 | 442.69 | 1015.54 | 12.25 | 13.71 | 14.06 | 2931.06 | 5124.89 | 6513.32 | ✅ | +| c8_i512_o256 | 8 | 512 | 256 | 52.73 | 160 | 0 | 3.03 | 1553.68 | 629.48 | 2183.16 | 135.86 | 271.88 | 289.07 | 10.80 | 11.89 | 12.07 | 2369.82 | 3039.64 | 3126.47 | ✅ | +| c128_i16384_o1024 | 128 | 16384 | 1024 | 1401.24 | 2560 | 0 | 1.83 | 29932.73 | 453.17 | 30385.90 | 6827.03 | 13719.55 | 53845.73 | 254.25 | 266.59 | 267.01 | 69085.30 | 135447.23 | 185844.63 | ❌ | +| c128_i16384_o2048 | 128 | 16384 | 2048 | 1400.62 | 2560 | 0 | 1.83 | 29945.97 | 468.45 | 30414.41 | 5874.33 | 13475.14 | 53934.11 | 249.38 | 266.26 | 266.71 | 69277.40 | 141505.20 | 189264.63 | ❌ | +| c1_i131072_o128 | 1 | 131072 | 128 | 182.19 | 20 | 0 | 0.11 | 14388.29 | 12.72 | 14401.01 | 5392.41 | 5419.10 | 5420.54 | 6.96 | 6.97 | 6.98 | 6192.49 | 6221.42 | 6221.43 | ⚠️ | +| c1_i131072_o256 | 1 | 131072 | 256 | 195.04 | 20 | 0 | 0.10 | 13440.21 | 19.01 | 13459.22 | 5397.54 | 5431.66 | 5440.22 | 7.00 | 7.03 | 7.03 | 6686.72 | 7103.29 | 7119.59 | ⚠️ | +| c1_i131072_o512 | 1 | 131072 | 512 | 201.48 | 20 | 0 | 0.10 | 13010.69 | 20.19 | 13030.87 | 5500.61 | 5555.72 | 5562.25 | 7.00 | 7.02 | 7.03 | 6917.34 | 7364.34 | 8109.96 | ⚠️ | +| c1_i262144_o256 | 1 | 262144 | 256 | 426.96 | 20 | 0 | 0.05 | 12279.65 | 8.19 | 12287.84 | 13218.70 | 13504.54 | 13586.50 | 7.22 | 7.32 | 7.36 | 14470.75 | 14834.79 | 14835.15 | ⚠️ | +| c1_i524288_o128 | 1 | 524288 | 128 | 1098.28 | 20 | 0 | 0.02 | 9547.46 | 1.62 | 9549.08 | 36239.34 | 36595.44 | 37102.25 | 9.37 | 9.80 | 37.00 | 36907.34 | 37308.40 | 37807.76 | ⚠️ | +| c1_i65536_o1024 | 1 | 65536 | 1024 | 114.83 | 20 | 0 | 0.17 | 11414.14 | 39.37 | 11453.51 | 2462.78 | 2475.26 | 2495.15 | 6.92 | 6.93 | 6.93 | 4020.21 | 5428.03 | 7058.64 | ✅ | +| c1_i65536_o128 | 1 | 65536 | 128 | 96.37 | 20 | 0 | 0.21 | 13600.71 | 24.89 | 13625.61 | 2497.10 | 2528.43 | 2558.93 | 6.87 | 6.87 | 6.88 | 3313.69 | 3344.68 | 3373.65 | ✅ | +| c1_i65536_o256 | 1 | 65536 | 256 | 105.96 | 20 | 0 | 0.19 | 12370.49 | 32.66 | 12403.15 | 2460.46 | 2472.33 | 2483.17 | 6.92 | 6.93 | 6.93 | 3650.21 | 4159.14 | 4159.75 | ✅ | +| c1_i65536_o512 | 1 | 65536 | 512 | 108.81 | 20 | 0 | 0.18 | 12045.94 | 34.57 | 12080.52 | 2461.82 | 2468.77 | 2498.35 | 6.92 | 6.93 | 6.93 | 3756.97 | 4315.50 | 4593.51 | ✅ | +| c32_i65536_o1024 | 32 | 65536 | 1024 | 395.45 | 160 | 0 | 0.40 | 26515.91 | 89.22 | 26605.13 | 18530.86 | 54451.46 | 68783.28 | 252.00 | 283.71 | 284.41 | 72731.85 | 117936.92 | 151338.20 | ❌ | +| c8_i131072_o512 | 8 | 131072 | 512 | 887.91 | 160 | 0 | 0.18 | 23619.01 | 33.13 | 23652.14 | 12820.61 | 25478.95 | 32945.62 | 156.62 | 223.19 | 294.51 | 41479.53 | 66809.84 | 79279.57 | ❌ | +| c8_i262144_o128 | 8 | 262144 | 128 | 2121.21 | 160 | 0 | 0.08 | 19773.16 | 9.48 | 19782.64 | 51391.65 | 61587.85 | 80661.75 | 382.04 | 388.68 | 389.24 | 99110.05 | 110875.08 | 129961.05 | ❌ | +| c8_i65536_o1024 | 8 | 65536 | 1024 | 422.41 | 160 | 0 | 0.38 | 24823.70 | 76.89 | 24900.59 | 4665.85 | 8696.15 | 14818.98 | 73.86 | 99.00 | 113.60 | 19699.15 | 35998.21 | 56237.93 | ❌ | +| c8_i65536_o128 | 8 | 65536 | 128 | 402.35 | 160 | 0 | 0.40 | 26061.00 | 46.48 | 26107.48 | 8659.91 | 13193.45 | 15765.71 | 87.87 | 143.78 | 166.11 | 18805.63 | 21397.73 | 22483.11 | ❌ | +| c8_i65536_o256 | 8 | 65536 | 256 | 416.75 | 160 | 0 | 0.38 | 25160.80 | 67.25 | 25228.06 | 5113.03 | 10698.95 | 14785.06 | 82.53 | 113.13 | 132.88 | 19445.90 | 27248.32 | 32918.89 | ❌ | +| c8_i65536_o512 | 8 | 65536 | 512 | 422.66 | 160 | 0 | 0.38 | 24809.03 | 76.23 | 24885.26 | 4532.43 | 9015.11 | 14789.72 | 75.30 | 108.22 | 126.16 | 19689.27 | 34072.42 | 42942.27 | ❌ | + +SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail. + diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json new file mode 100644 index 0000000..7c20a16 --- /dev/null +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/results.json @@ -0,0 +1,6428 @@ +{ + "metadata": { + "experiment": "dsv4_h200_vllm_tp8_custom_bench", + "run_id": "20260711-193756", + "timestamp": "2026-07-11T19:37:56+00:00", + "model": "/data/models/DeepSeek-V4-Flash", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H200 143GB", + "accelerator": "NVIDIA H200", + "chip": "nvidia_h200", + "script": "experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh", + "env": "/data/user1/yy/envs/vllm", + "git_commit": "def1355", + "git_dirty": "dirty", + "description": "H200 1x vLLM TP=8 single-service benchmark using bench_client.py" + }, + "config": { + "tp": 8, + "num_services": 1, + "ports": "30005", + "gpu_groups": "0,1,2,3,4,5,6,7", + "kv_cache_dtype": "fp8", + "block_size": 256, + "client": "bench_client.py", + "lb_strategy": "round_robin", + "scenarios": [ + "1 512 256 20", + "2 512 256 40", + "4 512 256 80", + "8 512 256 160", + "16 512 256 320", + "32 512 256 640", + "64 512 256 1280", + "128 512 256 2560", + "1 2048 512 20", + "8 2048 512 160", + "32 2048 512 640", + "128 2048 512 2560", + "1 4096 1024 20", + "8 4096 1024 160", + "32 4096 1024 640", + "128 4096 1024 2560" + ] + }, + "scenarios": [ + { + "name": "c128_i1024_o1024", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 206.74895269901026, + "request_throughput": 12.38216671272287, + "input_token_throughput": 12679.33871382822, + "output_token_throughput": 2588.491951294716, + "total_token_throughput": 15267.830665122934, + "total_input_tokens": 2621440, + "total_output_tokens": 535168, + "e2e_ms": { + "mean": 9995.612086433426, + "p50": 10042.748603475047, + "p90": 12876.024052471621, + "p95": 13713.982918864353, + "p99": 15284.945209203279 + }, + "ttft_ms": { + "mean": 355.5099342091353, + "p50": 243.9829669892788, + "p90": 386.2356346042361, + "p95": 540.9974577196409, + "p99": 3399.252886212198 + }, + "tpot_ms": { + "mean": 46.34608583039063, + "p50": 47.0857676038808, + "p90": 49.548431669654995, + "p95": 50.57579517885789, + "p99": 52.541381539005876 + }, + "itl_ms": { + "mean": 46.11291511528496, + "p50": 46.881382113519905, + "p90": 49.288727774137215, + "p95": 50.29987956637689, + "p99": 52.29071026166347 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_1024.jsonl" + }, + { + "name": "c128_i1024_o128", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 133.00950148300035, + "request_throughput": 19.246745318620626, + "input_token_throughput": 19708.66720626752, + "output_token_throughput": 2459.6964604202653, + "total_token_throughput": 22168.363666687786, + "total_input_tokens": 2621440, + "total_output_tokens": 327163, + "e2e_ms": { + "mean": 6458.710522773936, + "p50": 6450.677961984184, + "p90": 6679.613613686524, + "p95": 7012.215024293982, + "p99": 8759.515378992073 + }, + "ttft_ms": { + "mean": 1183.3169419023761, + "p50": 1213.7640539731365, + "p90": 1461.3176107173786, + "p95": 1479.2902125336695, + "p99": 3426.818067198619 + }, + "tpot_ms": { + "mean": 41.60377345263331, + "p50": 41.42248976997854, + "p90": 45.77268147799847, + "p95": 47.49906829926808, + "p99": 49.21793915747983 + }, + "itl_ms": { + "mean": 41.60175583400803, + "p50": 41.42150920233689, + "p90": 45.77246147393997, + "p95": 47.49885068504657, + "p99": 49.21768283782669 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_128.jsonl" + }, + { + "name": "c128_i1024_o2048", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.80468665598892, + "request_throughput": 12.260261208684772, + "input_token_throughput": 12554.507477693207, + "output_token_throughput": 2547.1602602304197, + "total_token_throughput": 15101.667737923628, + "total_input_tokens": 2621440, + "total_output_tokens": 531859, + "e2e_ms": { + "mean": 10106.575596190418, + "p50": 10063.31227402552, + "p90": 13078.549599891994, + "p95": 13980.859304094341, + "p99": 15725.898965689119 + }, + "ttft_ms": { + "mean": 351.4643375147216, + "p50": 240.56834846851416, + "p90": 374.20819198014215, + "p95": 493.9341375400565, + "p99": 3429.205895262421 + }, + "tpot_ms": { + "mean": 47.17323325059421, + "p50": 47.75218639386304, + "p90": 49.633491854371265, + "p95": 52.36074024377563, + "p99": 59.290788935476655 + }, + "itl_ms": { + "mean": 46.937751244996235, + "p50": 47.51707957709188, + "p90": 49.415206330399855, + "p95": 51.937400787877415, + "p99": 59.03572046826236 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_2048.jsonl" + }, + { + "name": "c128_i1024_o256", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 205.11042719194666, + "request_throughput": 12.481081703390428, + "input_token_throughput": 12780.627664271798, + "output_token_throughput": 2561.2349756766853, + "total_token_throughput": 15341.862639948484, + "total_input_tokens": 2621440, + "total_output_tokens": 525336, + "e2e_ms": { + "mean": 9935.310300847526, + "p50": 10115.252760995645, + "p90": 12385.100717213936, + "p95": 12606.547662557568, + "p99": 13028.91704150126 + }, + "ttft_ms": { + "mean": 351.2175043693105, + "p50": 243.14397198031656, + "p90": 373.0044780881144, + "p95": 456.94061022368277, + "p99": 3426.5382378635695 + }, + "tpot_ms": { + "mean": 46.93347030558093, + "p50": 47.9216824371189, + "p90": 49.75475172500139, + "p95": 50.35507099599905, + "p99": 51.38505285855793 + }, + "itl_ms": { + "mean": 46.72323950761121, + "p50": 47.723609778449614, + "p90": 49.567892764403894, + "p95": 50.12071192187135, + "p99": 51.17095904803956 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_256.jsonl" + }, + { + "name": "c128_i1024_o4096", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 208.6310452229809, + "request_throughput": 12.270465295631917, + "input_token_throughput": 12564.956462727083, + "output_token_throughput": 2562.1786030390786, + "total_token_throughput": 15127.13506576616, + "total_input_tokens": 2621440, + "total_output_tokens": 534550, + "e2e_ms": { + "mean": 10079.333489348573, + "p50": 10091.16335248109, + "p90": 12926.991533790713, + "p95": 13938.759127602681, + "p99": 15626.708482055688 + }, + "ttft_ms": { + "mean": 352.13409095886163, + "p50": 244.19353948906064, + "p90": 370.4628453939222, + "p95": 475.0491648941544, + "p99": 3401.064061677898 + }, + "tpot_ms": { + "mean": 46.82233196854027, + "p50": 47.89751414392755, + "p90": 49.819039048612304, + "p95": 50.3518367372333, + "p99": 51.5907629143145 + }, + "itl_ms": { + "mean": 46.58335775413696, + "p50": 47.644176474945255, + "p90": 49.59239070185347, + "p95": 50.0927766120728, + "p99": 51.20594759062769 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_4096.jsonl" + }, + { + "name": "c128_i1024_o512", + "config": { + "concurrency": 128, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 210.2132963129552, + "request_throughput": 12.178106927112726, + "input_token_throughput": 12470.381493363431, + "output_token_throughput": 2534.3829783576184, + "total_token_throughput": 15004.76447172105, + "total_input_tokens": 2621440, + "total_output_tokens": 532761, + "e2e_ms": { + "mean": 10138.544655314876, + "p50": 10175.040662987158, + "p90": 13069.769149302738, + "p95": 13931.679938701564, + "p99": 15398.916205506535 + }, + "ttft_ms": { + "mean": 355.65935904862727, + "p50": 240.44733747723512, + "p90": 393.7936486501712, + "p95": 519.455098820618, + "p99": 3397.328862186987 + }, + "tpot_ms": { + "mean": 47.251763059437835, + "p50": 47.96887544839744, + "p90": 50.820445244109074, + "p95": 52.83991169163644, + "p99": 55.04587439127752 + }, + "itl_ms": { + "mean": 47.00573781106416, + "p50": 47.71712084654526, + "p90": 50.64945889698164, + "p95": 52.655936409592336, + "p99": 54.68855691765819 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_1024_512.jsonl" + }, + { + "name": "c128_i16384_o128", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1359.066230058961, + "request_throughput": 1.883646244295937, + "input_token_throughput": 30861.66006654463, + "output_token_throughput": 240.90584605710933, + "total_token_throughput": 31102.56591260174, + "total_input_tokens": 41943040, + "total_output_tokens": 327407, + "e2e_ms": { + "mean": 67037.50906514256, + "p50": 67357.02024202328, + "p90": 67636.39427277376, + "p95": 67882.99461098795, + "p99": 86482.76843836233 + }, + "ttft_ms": { + "mean": 34308.65381123959, + "p50": 34204.28299551713, + "p90": 34482.4229988968, + "p95": 34726.7470557912, + "p99": 53850.1370839576 + }, + "tpot_ms": { + "mean": 257.92863620986134, + "p50": 261.05848694491516, + "p90": 261.1143110553719, + "p95": 261.1287856930326, + "p99": 261.24581678764065 + }, + "itl_ms": { + "mean": 257.9097102152317, + "p50": 261.05787166541876, + "p90": 261.1123652793247, + "p95": 261.1257119797258, + "p99": 261.14682886486884 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_128.jsonl" + }, + { + "name": "c128_i16384_o256", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1383.2075693949591, + "request_throughput": 1.850770670030234, + "input_token_throughput": 30323.026657775354, + "output_token_throughput": 393.1159805884024, + "total_token_throughput": 30716.14263836376, + "total_input_tokens": 41943040, + "total_output_tokens": 543761, + "e2e_ms": { + "mean": 68522.27549761867, + "p50": 68875.66710749525, + "p90": 80996.00835689344, + "p95": 82348.9028138516, + "p99": 113143.29638896219 + }, + "ttft_ms": { + "mean": 13674.308820916713, + "p50": 12427.863965480356, + "p90": 15051.10284507391, + "p95": 16619.003294425784, + "p99": 53959.96751072168 + }, + "tpot_ms": { + "mean": 259.7002911860435, + "p50": 265.0982799215538, + "p90": 265.59337430180994, + "p95": 265.74050924260996, + "p99": 266.01097282252385 + }, + "itl_ms": { + "mean": 258.7413562117342, + "p50": 263.88649323717783, + "p90": 264.0145869704262, + "p95": 264.049955442955, + "p99": 264.0919490046659 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_256.jsonl" + }, + { + "name": "c128_i16384_o512", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1394.5807457869523, + "request_throughput": 1.8356771436389003, + "input_token_throughput": 30075.734321379743, + "output_token_throughput": 449.1364174446214, + "total_token_throughput": 30524.870738824364, + "total_input_tokens": 41943040, + "total_output_tokens": 626357, + "e2e_ms": { + "mean": 68997.86024277919, + "p50": 61218.58670547954, + "p90": 111877.90824500262, + "p95": 134484.1842920287, + "p99": 142208.8375359383 + }, + "ttft_ms": { + "mean": 6929.190793514613, + "p50": 5308.056419016793, + "p90": 8465.563770505833, + "p95": 14006.295557459864, + "p99": 53929.18027237865 + }, + "tpot_ms": { + "mean": 256.42045890030613, + "p50": 265.1347581875615, + "p90": 266.20946280775973, + "p95": 266.3879577864928, + "p99": 266.67562587471275 + }, + "itl_ms": { + "mean": 255.26895726498742, + "p50": 264.1093867197653, + "p90": 264.64687804993525, + "p95": 264.6946616676973, + "p99": 264.82593538929 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_16384_512.jsonl" + }, + { + "name": "c128_i2048_o1024", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 274.182873593003, + "request_throughput": 9.336834086143773, + "input_token_throughput": 19121.836208422446, + "output_token_throughput": 2068.13428048655, + "total_token_throughput": 21189.970488908995, + "total_input_tokens": 5242880, + "total_output_tokens": 567047, + "e2e_ms": { + "mean": 13253.984380322005, + "p50": 12231.172265019268, + "p90": 18871.34518827661, + "p95": 21801.421404152643, + "p99": 28048.55546446866 + }, + "ttft_ms": { + "mean": 541.546650899204, + "p50": 335.68961051059887, + "p90": 544.6287516097073, + "p95": 764.6706944942717, + "p99": 6604.665919882827 + }, + "tpot_ms": { + "mean": 57.69224896148207, + "p50": 58.662424903217726, + "p90": 61.13828027834164, + "p95": 62.17687225050866, + "p99": 65.53232212942441 + }, + "itl_ms": { + "mean": 57.41329839820827, + "p50": 58.421607822445225, + "p90": 60.78241045852147, + "p95": 61.94631333915732, + "p99": 65.1391069201185 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_1024.jsonl" + }, + { + "name": "c128_i2048_o128", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 212.99189165403368, + "request_throughput": 12.019236883243664, + "input_token_throughput": 24615.397136883024, + "output_token_throughput": 1530.0206851504734, + "total_token_throughput": 26145.417822033498, + "total_input_tokens": 5242880, + "total_output_tokens": 325882, + "e2e_ms": { + "mean": 10431.321765192843, + "p50": 10470.64316298929, + "p90": 11085.870366205927, + "p95": 11370.95141907048, + "p99": 15884.139941605354 + }, + "ttft_ms": { + "mean": 1293.090087216433, + "p50": 1246.8626230256632, + "p90": 1747.983089205809, + "p95": 1997.4948156188475, + "p99": 6703.310411417624 + }, + "tpot_ms": { + "mean": 72.32697406863772, + "p50": 73.10035818496601, + "p90": 78.97615628596579, + "p95": 80.38428567705573, + "p99": 81.7834753297109 + }, + "itl_ms": { + "mean": 72.29165904992381, + "p50": 73.09435472038106, + "p90": 78.97258139450074, + "p95": 80.38391650942664, + "p99": 81.7239847918815 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_128.jsonl" + }, + { + "name": "c128_i2048_o2048", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 272.0649238389451, + "request_throughput": 9.409518742355221, + "input_token_throughput": 19270.694384343493, + "output_token_throughput": 2055.1601879079185, + "total_token_throughput": 21325.854572251414, + "total_input_tokens": 5242880, + "total_output_tokens": 559137, + "e2e_ms": { + "mean": 13143.514178946545, + "p50": 12082.19302649377, + "p90": 18679.455636080816, + "p95": 21512.404798675576, + "p99": 28583.361954419513 + }, + "ttft_ms": { + "mean": 542.6618820346221, + "p50": 339.305431523826, + "p90": 539.0634975978171, + "p95": 696.4638119126903, + "p99": 6595.59385088447 + }, + "tpot_ms": { + "mean": 58.015877285501915, + "p50": 59.11393804880664, + "p90": 61.71648090264369, + "p95": 62.670298319004864, + "p99": 65.22564691015543 + }, + "itl_ms": { + "mean": 57.71674817147152, + "p50": 58.78867086320406, + "p90": 61.42682309865147, + "p95": 62.303509328181136, + "p99": 64.79217375997129 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_2048.jsonl" + }, + { + "name": "c128_i2048_o256", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 263.82883767504245, + "request_throughput": 9.70326072979614, + "input_token_throughput": 19872.277974622495, + "output_token_throughput": 1986.560698287076, + "total_token_throughput": 21858.838672909573, + "total_input_tokens": 5242880, + "total_output_tokens": 524112, + "e2e_ms": { + "mean": 12841.303998676336, + "p50": 12770.766553498106, + "p90": 16183.979681675555, + "p95": 16454.99841661367, + "p99": 19770.974340796816 + }, + "ttft_ms": { + "mean": 552.7428740571622, + "p50": 338.345098978607, + "p90": 574.8256633232812, + "p95": 766.9361727079388, + "p99": 6667.41566293058 + }, + "tpot_ms": { + "mean": 60.36128601226305, + "p50": 61.53765959324275, + "p90": 64.03844464794237, + "p95": 64.80210034227288, + "p99": 67.6115024987387 + }, + "itl_ms": { + "mean": 60.09187703963762, + "p50": 61.29940147707798, + "p90": 63.74183259078168, + "p95": 64.42361930701372, + "p99": 67.1758936519173 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_256.jsonl" + }, + { + "name": "c128_i2048_o4096", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 274.1830678950064, + "request_throughput": 9.336827469522325, + "input_token_throughput": 19121.82265758172, + "output_token_throughput": 2075.5913352677335, + "total_token_throughput": 21197.41399284945, + "total_input_tokens": 5242880, + "total_output_tokens": 569092, + "e2e_ms": { + "mean": 13271.956057284842, + "p50": 12218.301199522102, + "p90": 19085.673499788387, + "p95": 22132.755441660993, + "p99": 28679.850054419945 + }, + "ttft_ms": { + "mean": 538.8494964908205, + "p50": 329.8501919780392, + "p90": 536.288884928217, + "p95": 772.7042476995854, + "p99": 6666.243869229802 + }, + "tpot_ms": { + "mean": 57.59475360987989, + "p50": 58.62241071854672, + "p90": 60.865193869045825, + "p95": 61.732659389700814, + "p99": 65.98072567714664 + }, + "itl_ms": { + "mean": 57.31168219239646, + "p50": 58.35251099982605, + "p90": 60.52328757509878, + "p95": 61.41718757690342, + "p99": 65.77234831641486 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_4096.jsonl" + }, + { + "name": "c128_i2048_o512", + "config": { + "concurrency": 128, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 271.79637533798814, + "request_throughput": 9.418815820544156, + "input_token_throughput": 19289.73480047443, + "output_token_throughput": 2074.255034854407, + "total_token_throughput": 21363.989835328837, + "total_input_tokens": 5242880, + "total_output_tokens": 563775, + "e2e_ms": { + "mean": 13185.11682162771, + "p50": 12167.376212484669, + "p90": 18797.917102725478, + "p95": 21822.20230518142, + "p99": 28335.43754096433 + }, + "ttft_ms": { + "mean": 544.2361912605747, + "p50": 335.83095151698217, + "p90": 560.1696573663503, + "p95": 780.7416238734743, + "p99": 6725.671748666791 + }, + "tpot_ms": { + "mean": 57.75487659402953, + "p50": 58.80900923345935, + "p90": 60.99471152181436, + "p95": 61.62449590523302, + "p99": 65.39932975583538 + }, + "itl_ms": { + "mean": 57.481931975649466, + "p50": 58.51227060084552, + "p90": 60.6782899592584, + "p95": 61.29368894317035, + "p99": 65.19139830356515 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_2048_512.jsonl" + }, + { + "name": "c128_i4096_o1024", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 424.82515185599914, + "request_throughput": 6.026008556263049, + "input_token_throughput": 24682.53104645345, + "output_token_throughput": 1273.8087602295016, + "total_token_throughput": 25956.33980668295, + "total_input_tokens": 10485760, + "total_output_tokens": 541146, + "e2e_ms": { + "mean": 20697.6679702115, + "p50": 19206.427480472485, + "p90": 29670.110822987044, + "p95": 35507.95289489034, + "p99": 47170.14222518534 + }, + "ttft_ms": { + "mean": 1014.2606450839367, + "p50": 589.3256725103129, + "p90": 1090.3332160029092, + "p95": 1488.9069345052123, + "p99": 13325.899195196107 + }, + "tpot_ms": { + "mean": 93.93489520852809, + "p50": 95.33726487295453, + "p90": 101.10578242438245, + "p95": 103.74371460626769, + "p99": 115.23677950752673 + }, + "itl_ms": { + "mean": 93.4331025955482, + "p50": 94.93783975657502, + "p90": 100.51129917148953, + "p95": 103.0640243406409, + "p99": 114.33476458414724 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_1024.jsonl" + }, + { + "name": "c128_i4096_o128", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 371.4178167580394, + "request_throughput": 6.892507264043596, + "input_token_throughput": 28231.70975352257, + "output_token_throughput": 873.7060672870264, + "total_token_throughput": 29105.415820809598, + "total_input_tokens": 10485760, + "total_output_tokens": 324510, + "e2e_ms": { + "mean": 18310.57876672269, + "p50": 18448.31012902432, + "p90": 19225.236213795142, + "p95": 19462.368021774455, + "p99": 30169.283645756623 + }, + "ttft_ms": { + "mean": 1687.7391880757386, + "p50": 1529.599169472931, + "p90": 2042.0844946987927, + "p95": 2056.5731700480683, + "p99": 13344.424836942344 + }, + "tpot_ms": { + "mean": 132.0821975364293, + "p50": 133.9608105825348, + "p90": 140.01268617562238, + "p95": 142.1111826331164, + "p99": 145.57791828146406 + }, + "itl_ms": { + "mean": 131.99716351450823, + "p50": 133.95380003535462, + "p90": 140.00703786379717, + "p95": 142.11082250321206, + "p99": 145.07502242550618 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_128.jsonl" + }, + { + "name": "c128_i4096_o2048", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 425.2518261469668, + "request_throughput": 6.019962390744127, + "input_token_throughput": 24657.765952487945, + "output_token_throughput": 1288.0344452905458, + "total_token_throughput": 25945.80039777849, + "total_input_tokens": 10485760, + "total_output_tokens": 547739, + "e2e_ms": { + "mean": 20779.354470113663, + "p50": 19269.694546994288, + "p90": 29949.315433588345, + "p95": 36358.763673005196, + "p99": 50447.49927988618 + }, + "ttft_ms": { + "mean": 1019.4579501709768, + "p50": 598.1724759622011, + "p90": 1096.0077470983379, + "p95": 1464.8891508928511, + "p99": 13332.857389408746 + }, + "tpot_ms": { + "mean": 93.12786586783068, + "p50": 94.68724737436233, + "p90": 99.93479914307919, + "p95": 101.5232748251677, + "p99": 111.69156290842999 + }, + "itl_ms": { + "mean": 92.65871392345448, + "p50": 94.17968940964437, + "p90": 99.46187302676536, + "p95": 101.09497997720692, + "p99": 110.93734985374068 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_2048.jsonl" + }, + { + "name": "c128_i4096_o256", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 413.8003815539996, + "request_throughput": 6.186557852813213, + "input_token_throughput": 25340.14096512292, + "output_token_throughput": 1217.7924972121846, + "total_token_throughput": 26557.933462335102, + "total_input_tokens": 10485760, + "total_output_tokens": 503923, + "e2e_ms": { + "mean": 20311.651262024407, + "p50": 20182.22546146717, + "p90": 26356.96079552872, + "p95": 26976.595176249975, + "p99": 33054.57394192112 + }, + "ttft_ms": { + "mean": 1065.7509551069552, + "p50": 660.4246550123207, + "p90": 1217.6136810914613, + "p95": 1554.3095287983313, + "p99": 13333.887037970126 + }, + "tpot_ms": { + "mean": 98.37467601072895, + "p50": 100.35475919842096, + "p90": 106.18230888031277, + "p95": 108.53097404087819, + "p99": 114.97146971749977 + }, + "itl_ms": { + "mean": 97.90174442685736, + "p50": 100.01569229517985, + "p90": 105.46145688671882, + "p95": 107.86138331901421, + "p99": 114.5162470124037 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_256.jsonl" + }, + { + "name": "c128_i4096_o4096", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 423.761194480001, + "request_throughput": 6.041138342413316, + "input_token_throughput": 24744.50265052494, + "output_token_throughput": 1282.561987930327, + "total_token_throughput": 26027.064638455267, + "total_input_tokens": 10485760, + "total_output_tokens": 543500, + "e2e_ms": { + "mean": 20723.11629090143, + "p50": 19256.915692501934, + "p90": 29976.309534144817, + "p95": 35255.544051312616, + "p99": 48374.307467846615 + }, + "ttft_ms": { + "mean": 1040.8386572691825, + "p50": 606.4721180300694, + "p90": 1181.8239250802435, + "p95": 1941.310445405544, + "p99": 13328.937759977998 + }, + "tpot_ms": { + "mean": 93.33886441284803, + "p50": 94.96167027747198, + "p90": 99.94806040223267, + "p95": 101.80580796030549, + "p99": 111.130208978832 + }, + "itl_ms": { + "mean": 92.84261079498634, + "p50": 94.58125016055442, + "p90": 99.3758299306611, + "p95": 101.0630073622523, + "p99": 110.63285175462228 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_4096.jsonl" + }, + { + "name": "c128_i4096_o512", + "config": { + "concurrency": 128, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 423.6984583950252, + "request_throughput": 6.042032840282946, + "input_token_throughput": 24748.166513798948, + "output_token_throughput": 1274.4157768366804, + "total_token_throughput": 26022.58229063563, + "total_input_tokens": 10485760, + "total_output_tokens": 539968, + "e2e_ms": { + "mean": 20669.59312196584, + "p50": 19350.259740982438, + "p90": 29572.45529227657, + "p95": 35396.549672255096, + "p99": 49034.77441504888 + }, + "ttft_ms": { + "mean": 1032.5226390652233, + "p50": 598.7296614912339, + "p90": 1154.9013265117537, + "p95": 1562.556813692209, + "p99": 13323.3382030559 + }, + "tpot_ms": { + "mean": 93.86471050008637, + "p50": 95.59369101226693, + "p90": 101.53613190663701, + "p95": 103.68156125609973, + "p99": 112.50945685705346 + }, + "itl_ms": { + "mean": 93.36493755809302, + "p50": 95.09812282953814, + "p90": 100.97701147581158, + "p95": 103.27911349553783, + "p99": 111.96191368855567 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_4096_512.jsonl" + }, + { + "name": "c128_i512_o256", + "config": { + "concurrency": 128, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 198.04905138298636, + "request_throughput": 12.926090693812432, + "input_token_throughput": 6618.158435231965, + "output_token_throughput": 2673.832549573583, + "total_token_throughput": 9291.990984805549, + "total_input_tokens": 1310720, + "total_output_tokens": 529550, + "e2e_ms": { + "mean": 9585.923150789631, + "p50": 10002.822702517733, + "p90": 12147.832186403684, + "p95": 13135.335343127372, + "p99": 14237.255100695762 + }, + "ttft_ms": { + "mean": 290.6632589968467, + "p50": 214.245242474135, + "p90": 345.00691113644274, + "p95": 500.99967721907893, + "p99": 1944.8683028796222 + }, + "tpot_ms": { + "mean": 45.15565344448567, + "p50": 44.83139648013622, + "p90": 51.02216221175303, + "p95": 52.67646848178537, + "p99": 57.18692985139731 + }, + "itl_ms": { + "mean": 44.973391207111554, + "p50": 44.674817022825394, + "p90": 50.939215799513484, + "p95": 52.50806826338534, + "p99": 56.95803494225427 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_128_512_256.jsonl" + }, + { + "name": "c16_i512_o256", + "config": { + "concurrency": 16, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 320 + }, + "metrics": { + "success": 320, + "failed": 0, + "duration_s": 70.00084047595738, + "request_throughput": 4.5713736838618075, + "input_token_throughput": 2340.5433261372455, + "output_token_throughput": 954.2456854206282, + "total_token_throughput": 3294.789011557874, + "total_input_tokens": 163840, + "total_output_tokens": 66798, + "e2e_ms": { + "mean": 3227.912786142042, + "p50": 3303.816789004486, + "p90": 4077.1741356002167, + "p95": 4110.177500350983, + "p99": 4253.23702479538 + }, + "ttft_ms": { + "mean": 155.9941838546365, + "p50": 120.02038149512373, + "p90": 209.794121485902, + "p95": 299.16939962713525, + "p99": 421.0051234619459 + }, + "tpot_ms": { + "mean": 14.771496620725333, + "p50": 14.982428883652467, + "p90": 16.07892679411306, + "p95": 16.431581466333803, + "p99": 17.43982213951822 + }, + "itl_ms": { + "mean": 14.709423569418314, + "p50": 14.96633889086642, + "p90": 15.991478397076191, + "p95": 16.343193537253118, + "p99": 17.36775174975628 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_16_512_256.jsonl" + }, + { + "name": "c1_i1024_o1024", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.001702999987174, + "request_throughput": 0.48778461714154303, + "input_token_throughput": 499.49144795294006, + "output_token_throughput": 102.02015267515372, + "total_token_throughput": 601.5116006280938, + "total_input_tokens": 20480, + "total_output_tokens": 4183, + "e2e_ms": { + "mean": 1513.646431043162, + "p50": 1574.0484629932325, + "p90": 1819.820330955554, + "p95": 1949.2676253576064, + "p99": 1998.6603202793049 + }, + "ttft_ms": { + "mean": 113.2725309493253, + "p50": 111.85279348865151, + "p90": 113.1193755834829, + "p95": 116.29407556611115, + "p99": 137.7853335009422 + }, + "tpot_ms": { + "mean": 6.728842661121109, + "p50": 6.728117721421029, + "p90": 6.73756308164125, + "p95": 6.738021995986233, + "p99": 6.7388486460154216 + }, + "itl_ms": { + "mean": 6.692791953543756, + "p50": 6.693387171586178, + "p90": 6.6951132749056494, + "p95": 6.695167897476331, + "p99": 6.695272520098379 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_1024.jsonl" + }, + { + "name": "c1_i1024_o128", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 26.62190206797095, + "request_throughput": 0.7512611213479814, + "input_token_throughput": 769.291388260333, + "output_token_throughput": 96.16142353254162, + "total_token_throughput": 865.4528117928745, + "total_input_tokens": 20480, + "total_output_tokens": 2560, + "e2e_ms": { + "mean": 961.8738458957523, + "p50": 961.338467488531, + "p90": 962.9271580080967, + "p95": 964.1112041892484, + "p99": 977.0850000297651 + }, + "ttft_ms": { + "mean": 112.09932600322645, + "p50": 111.56318802386522, + "p90": 112.80985138146207, + "p95": 114.09245460527019, + "p99": 127.54729093285275 + }, + "tpot_ms": { + "mean": 6.691137951909653, + "p50": 6.690667098384409, + "p90": 6.694461964299961, + "p95": 6.695022723819936, + "p99": 6.696434283411812 + }, + "itl_ms": { + "mean": 6.6905939176735485, + "p50": 6.69014484252172, + "p90": 6.6939145778067495, + "p95": 6.694489817490698, + "p99": 6.695905075294853 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_128.jsonl" + }, + { + "name": "c1_i1024_o2048", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.37348232802469, + "request_throughput": 0.49537465798726205, + "input_token_throughput": 507.26364977895634, + "output_token_throughput": 100.38767444111865, + "total_token_throughput": 607.651324220075, + "total_input_tokens": 20480, + "total_output_tokens": 4053, + "e2e_ms": { + "mean": 1470.1031726581277, + "p50": 1538.4048255218659, + "p90": 1877.3818529967684, + "p95": 1949.8724324192156, + "p99": 2093.403241672204 + }, + "ttft_ms": { + "mean": 113.53871540632099, + "p50": 112.32990000280552, + "p90": 115.95370600116439, + "p95": 118.16973473469263, + "p99": 129.37863651022778 + }, + "tpot_ms": { + "mean": 6.729301792871088, + "p50": 6.726905772857237, + "p90": 6.7408030649588095, + "p95": 6.749353885298162, + "p99": 6.749622846797336 + }, + "itl_ms": { + "mean": 6.6908831387070915, + "p50": 6.6918520466231985, + "p90": 6.694045093510076, + "p95": 6.694208334277694, + "p99": 6.694287661540632 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_2048.jsonl" + }, + { + "name": "c1_i1024_o256", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.66181249899091, + "request_throughput": 0.5042633893876849, + "input_token_throughput": 516.3657107329893, + "output_token_throughput": 101.40736760586343, + "total_token_throughput": 617.7730783388527, + "total_input_tokens": 20480, + "total_output_tokens": 4022, + "e2e_ms": { + "mean": 1457.720083554159, + "p50": 1424.1828475205693, + "p90": 1692.0858449302616, + "p95": 1819.5981201744871, + "p99": 1819.9626496067503 + }, + "ttft_ms": { + "mean": 112.03125940228347, + "p50": 111.31371150258929, + "p90": 112.13053134270012, + "p95": 113.19355736195577, + "p99": 128.9370090694865 + }, + "tpot_ms": { + "mean": 6.72637613476324, + "p50": 6.728691771932777, + "p90": 6.733196039297692, + "p95": 6.738730504584789, + "p99": 6.738919628883542 + }, + "itl_ms": { + "mean": 6.69216179570252, + "p50": 6.692514737256101, + "p90": 6.694407104034717, + "p95": 6.6958281427836415, + "p99": 6.697055611373656 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_256.jsonl" + }, + { + "name": "c1_i1024_o4096", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.069976897968445, + "request_throughput": 0.511902017557629, + "input_token_throughput": 524.1876659790121, + "output_token_throughput": 100.38398564305103, + "total_token_throughput": 624.5716516220631, + "total_input_tokens": 20480, + "total_output_tokens": 3922, + "e2e_ms": { + "mean": 1439.5377619453939, + "p50": 1474.0643894765526, + "p90": 1734.9704635154922, + "p95": 1773.7776646856219, + "p99": 1831.833771346137 + }, + "ttft_ms": { + "mean": 126.45561884564813, + "p50": 112.52622748725116, + "p90": 176.61038206424564, + "p95": 178.17131999472622, + "p99": 178.35113599256147 + }, + "tpot_ms": { + "mean": 6.731382327839597, + "p50": 6.729523633618532, + "p90": 6.741167108195672, + "p95": 6.742111598489635, + "p99": 6.7473587002338045 + }, + "itl_ms": { + "mean": 6.693141582456953, + "p50": 6.693129204125723, + "p90": 6.698133283139748, + "p95": 6.698879551499814, + "p99": 6.6989547085102945 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_4096.jsonl" + }, + { + "name": "c1_i1024_o512", + "config": { + "concurrency": 1, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 38.85685831902083, + "request_throughput": 0.5147096514030265, + "input_token_throughput": 527.0626830366991, + "output_token_throughput": 98.61836920881987, + "total_token_throughput": 625.6810522455189, + "total_input_tokens": 20480, + "total_output_tokens": 3832, + "e2e_ms": { + "mean": 1396.0474013991188, + "p50": 1421.305021009175, + "p90": 1761.8963810731657, + "p95": 1797.4548358935863, + "p99": 1820.3903575986624 + }, + "ttft_ms": { + "mean": 113.08509785449132, + "p50": 111.65635052020662, + "p90": 112.92990780784748, + "p95": 114.42024928692264, + "p99": 136.00369145569854 + }, + "tpot_ms": { + "mean": 6.7326943598063, + "p50": 6.7306789376316285, + "p90": 6.741523098781185, + "p95": 6.742240715060803, + "p99": 6.747612401371304 + }, + "itl_ms": { + "mean": 6.692899736605012, + "p50": 6.6925738048090775, + "p90": 6.695661536070422, + "p95": 6.695736518938616, + "p99": 6.696398640652917 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_1024_512.jsonl" + }, + { + "name": "c1_i16384_o1024", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 60.01604464603588, + "request_throughput": 0.33324422024071226, + "input_token_throughput": 5459.87330442383, + "output_token_throughput": 74.28013669165476, + "total_token_throughput": 5534.153441115484, + "total_input_tokens": 327680, + "total_output_tokens": 4458, + "e2e_ms": { + "mean": 2136.322955050855, + "p50": 1987.1548090013675, + "p90": 2866.87489899341, + "p95": 2962.6648157864115, + "p99": 3211.9196255604033 + }, + "ttft_ms": { + "mean": 621.8708320520818, + "p50": 618.9937324961647, + "p90": 631.5798229188658, + "p95": 633.1030654429924, + "p99": 637.2123010957148 + }, + "tpot_ms": { + "mean": 6.824606008095292, + "p50": 6.822637892302804, + "p90": 6.828144618603244, + "p95": 6.829834324474402, + "p99": 6.849728735808008 + }, + "itl_ms": { + "mean": 6.7858919799987145, + "p50": 6.784267973909232, + "p90": 6.802818123600203, + "p95": 6.806531345537709, + "p99": 6.810954561452206 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_1024.jsonl" + }, + { + "name": "c1_i16384_o128", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.67448142397916, + "request_throughput": 0.47990999087734687, + "input_token_throughput": 7862.845290534451, + "output_token_throughput": 60.87658234279145, + "total_token_throughput": 7923.721872877242, + "total_input_tokens": 327680, + "total_output_tokens": 2537, + "e2e_ms": { + "mean": 1480.094525500317, + "p50": 1480.2975574857555, + "p90": 1482.2705925849732, + "p95": 1482.5216800556518, + "p99": 1482.9828344029374 + }, + "ttft_ms": { + "mean": 628.48725815129, + "p50": 628.42066999292, + "p90": 630.9192391112447, + "p95": 631.5766573417932, + "p99": 636.3627922814339 + }, + "tpot_ms": { + "mean": 6.766875583214485, + "p50": 6.762159404590623, + "p90": 6.772095540245729, + "p95": 6.806862495630048, + "p99": 6.809469593646936 + }, + "itl_ms": { + "mean": 6.766333328781184, + "p50": 6.76163371018548, + "p90": 6.771564252188961, + "p95": 6.8063165623694655, + "p99": 6.808896829262376 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_128.jsonl" + }, + { + "name": "c1_i16384_o2048", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 63.797216865001246, + "request_throughput": 0.3134932992817101, + "input_token_throughput": 5136.274215431538, + "output_token_throughput": 80.75587389496853, + "total_token_throughput": 5217.030089326508, + "total_input_tokens": 327680, + "total_output_tokens": 5152, + "e2e_ms": { + "mean": 2381.452842548606, + "p50": 2055.640481004957, + "p90": 3254.1592128574866, + "p95": 4577.4711255711745, + "p99": 5209.664837909512 + }, + "ttft_ms": { + "mean": 630.1534300495405, + "p50": 629.4059840147384, + "p90": 633.5870493319817, + "p95": 636.3642237934982, + "p99": 636.9756711431546 + }, + "tpot_ms": { + "mean": 6.824520464142083, + "p50": 6.823017292777109, + "p90": 6.827681934593928, + "p95": 6.830904397375058, + "p99": 6.85009679783112 + }, + "itl_ms": { + "mean": 6.788338608930161, + "p50": 6.784363890505368, + "p90": 6.811085948498086, + "p95": 6.814371809257599, + "p99": 6.815686185589282 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_2048.jsonl" + }, + { + "name": "c1_i16384_o256", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 56.44182810600614, + "request_throughput": 0.35434713352014446, + "input_token_throughput": 5805.623435594047, + "output_token_throughput": 73.65105170216202, + "total_token_throughput": 5879.2744872962085, + "total_input_tokens": 327680, + "total_output_tokens": 4157, + "e2e_ms": { + "mean": 2040.2357441023923, + "p50": 2151.03490347974, + "p90": 2357.6732086134143, + "p95": 2358.859304583166, + "p99": 2362.4721705471165 + }, + "ttft_ms": { + "mean": 630.4795558040496, + "p50": 629.4021749927197, + "p90": 636.6133440285921, + "p95": 638.6065603641327, + "p99": 639.0439760766458 + }, + "tpot_ms": { + "mean": 6.816638198034302, + "p50": 6.8195045810406665, + "p90": 6.825905331977316, + "p95": 6.848790569941936, + "p99": 6.850559613468598 + }, + "itl_ms": { + "mean": 6.788158203215443, + "p50": 6.79268100974279, + "p90": 6.8146217625671985, + "p95": 6.818863007225994, + "p99": 6.8192545745722 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_256.jsonl" + }, + { + "name": "c1_i16384_o512", + "config": { + "concurrency": 1, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 70.93397790199379, + "request_throughput": 0.2819523251273611, + "input_token_throughput": 4619.506894886685, + "output_token_throughput": 85.86858061753783, + "total_token_throughput": 4705.375475504222, + "total_input_tokens": 327680, + "total_output_tokens": 6091, + "e2e_ms": { + "mean": 2692.1835272485623, + "p50": 2038.8584179745521, + "p90": 4093.8409490103368, + "p95": 4094.3862316198647, + "p99": 4094.9791175313294 + }, + "ttft_ms": { + "mean": 621.4529997989303, + "p50": 618.6122374783736, + "p90": 628.3789116656408, + "p95": 629.9593272589846, + "p99": 645.4197830701014 + }, + "tpot_ms": { + "mean": 6.823135096274382, + "p50": 6.820567188349704, + "p90": 6.8308570155862585, + "p95": 6.8461482453307925, + "p99": 6.851399853175462 + }, + "itl_ms": { + "mean": 6.793890181817107, + "p50": 6.794746884622608, + "p90": 6.814662841437065, + "p95": 6.814756591222776, + "p99": 6.814832595126425 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_16384_512.jsonl" + }, + { + "name": "c1_i2048_o1024", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.341512619983405, + "request_throughput": 0.46145136131632447, + "input_token_throughput": 945.0523879758325, + "output_token_throughput": 102.00382341897352, + "total_token_throughput": 1047.056211394806, + "total_input_tokens": 40960, + "total_output_tokens": 4421, + "e2e_ms": { + "mean": 1619.3287752947072, + "p50": 1505.6682314898353, + "p90": 2289.511943614343, + "p95": 2410.3983415960106, + "p99": 2903.386745112365 + }, + "ttft_ms": { + "mean": 123.14691609353758, + "p50": 121.50511099025607, + "p90": 123.98786658304743, + "p95": 127.94699885416777, + "p99": 148.69490539189425 + }, + "tpot_ms": { + "mean": 6.800605244175337, + "p50": 6.801867666012635, + "p90": 6.808000387354118, + "p95": 6.8084423111939865, + "p99": 6.809475911014002 + }, + "itl_ms": { + "mean": 6.764811909176386, + "p50": 6.765792570332284, + "p90": 6.7702946639625825, + "p95": 6.771773379877912, + "p99": 6.772758638576524 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_1024.jsonl" + }, + { + "name": "c1_i2048_o128", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 27.019237152009737, + "request_throughput": 0.7402133482703588, + "input_token_throughput": 1515.9569372576948, + "output_token_throughput": 94.11812723257611, + "total_token_throughput": 1610.075064490271, + "total_input_tokens": 40960, + "total_output_tokens": 2543, + "e2e_ms": { + "mean": 976.4232056535548, + "p50": 980.4443110188004, + "p90": 981.895874539623, + "p95": 983.6760145699373, + "p99": 1006.2214533070801 + }, + "ttft_ms": { + "mean": 122.84687320061494, + "p50": 121.22314149746671, + "p90": 123.54794240090996, + "p95": 126.64986727468205, + "p99": 147.4668486503651 + }, + "tpot_ms": { + "mean": 6.766764674291219, + "p50": 6.765895810960902, + "p90": 6.76797417273899, + "p95": 6.771030715730377, + "p99": 6.815647255658967 + }, + "itl_ms": { + "mean": 6.762888211734736, + "p50": 6.764920133919507, + "p90": 6.767030950639338, + "p95": 6.76742042006932, + "p99": 6.767525060283842 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_128.jsonl" + }, + { + "name": "c1_i2048_o2048", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.3504386190325, + "request_throughput": 0.46135634695099104, + "input_token_throughput": 944.8577985556296, + "output_token_throughput": 101.45226069452293, + "total_token_throughput": 1046.3100592501526, + "total_input_tokens": 40960, + "total_output_tokens": 4398, + "e2e_ms": { + "mean": 1611.8768365879077, + "p50": 1612.4180810002144, + "p90": 2009.0518668002917, + "p95": 2108.8781301688873, + "p99": 2496.0127236007243 + }, + "ttft_ms": { + "mean": 123.05733739340212, + "p50": 121.17295648204163, + "p90": 125.36015609512106, + "p95": 128.18555371195546, + "p99": 147.5431595003465 + }, + "tpot_ms": { + "mean": 6.80408578412217, + "p50": 6.801554977330165, + "p90": 6.812079547783474, + "p95": 6.829911392703745, + "p99": 6.835954278261701 + }, + "itl_ms": { + "mean": 6.766276811283118, + "p50": 6.767449128197479, + "p90": 6.771516465594735, + "p95": 6.771574771876405, + "p99": 6.771879487577742 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_2048.jsonl" + }, + { + "name": "c1_i2048_o256", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 37.216408888984006, + "request_throughput": 0.5373973630733611, + "input_token_throughput": 1100.5897995742434, + "output_token_throughput": 96.67778561689765, + "total_token_throughput": 1197.267585191141, + "total_input_tokens": 40960, + "total_output_tokens": 3598, + "e2e_ms": { + "mean": 1340.688644099282, + "p50": 1336.3160230219364, + "p90": 1525.1817746495365, + "p95": 1621.0493948310616, + "p99": 1775.5462661385534 + }, + "ttft_ms": { + "mean": 122.79568985104561, + "p50": 121.09551299363375, + "p90": 123.57838398893364, + "p95": 125.69494101044258, + "p99": 145.02478101814629 + }, + "tpot_ms": { + "mean": 6.808775584529611, + "p50": 6.8074119860722, + "p90": 6.816743647877187, + "p95": 6.8219026677205035, + "p99": 6.827699577873525 + }, + "itl_ms": { + "mean": 6.766320465357053, + "p50": 6.7664976447077425, + "p90": 6.769415223673198, + "p95": 6.769865434874809, + "p99": 6.770796189003204 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_256.jsonl" + }, + { + "name": "c1_i2048_o4096", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.114438192977104, + "request_throughput": 0.49857360344389495, + "input_token_throughput": 1021.0787398530969, + "output_token_throughput": 101.6093003818658, + "total_token_throughput": 1122.6880402349625, + "total_input_tokens": 40960, + "total_output_tokens": 4076, + "e2e_ms": { + "mean": 1502.661171098589, + "p50": 1435.3244435042143, + "p90": 2023.0349817953538, + "p95": 2076.606418809388, + "p99": 2137.6482325349934 + }, + "ttft_ms": { + "mean": 123.10377759858966, + "p50": 121.37787148822099, + "p90": 123.7979574885685, + "p95": 127.51484157342931, + "p99": 146.71904669667126 + }, + "tpot_ms": { + "mean": 6.803975411277669, + "p50": 6.804031260291214, + "p90": 6.812907689196067, + "p95": 6.816632568343603, + "p99": 6.817981062917237 + }, + "itl_ms": { + "mean": 6.765227059554421, + "p50": 6.766664099236885, + "p90": 6.770266656297683, + "p95": 6.7715020171936215, + "p99": 6.772113826350069 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_4096.jsonl" + }, + { + "name": "c1_i2048_o512", + "config": { + "concurrency": 1, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 37.019548388023395, + "request_throughput": 0.5402551049615295, + "input_token_throughput": 1106.4424549612124, + "output_token_throughput": 96.51657450137725, + "total_token_throughput": 1202.9590294625898, + "total_input_tokens": 40960, + "total_output_tokens": 3573, + "e2e_ms": { + "mean": 1331.89303005056, + "p50": 1333.1293719820678, + "p90": 1548.5706633073278, + "p95": 1558.9390544075286, + "p99": 1578.9031204831554 + }, + "ttft_ms": { + "mean": 122.79606895172037, + "p50": 121.24446599045768, + "p90": 123.67385803372599, + "p95": 127.3277018335648, + "p99": 146.09675077954304 + }, + "tpot_ms": { + "mean": 6.806607463414278, + "p50": 6.80578571430719, + "p90": 6.8121264678222015, + "p95": 6.817936847661569, + "p99": 6.8216227906584495 + }, + "itl_ms": { + "mean": 6.76395363464037, + "p50": 6.764548409562552, + "p90": 6.76793782551663, + "p95": 6.768732991808162, + "p99": 6.769085524112825 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_2048_512.jsonl" + }, + { + "name": "c1_i4096_o1024", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.121369317988865, + "request_throughput": 0.4638071637408934, + "input_token_throughput": 1899.7541426826995, + "output_token_throughput": 99.83449199522731, + "total_token_throughput": 1999.5886346779268, + "total_input_tokens": 81920, + "total_output_tokens": 4305, + "e2e_ms": { + "mean": 1644.17850044847, + "p50": 1627.3216654954012, + "p90": 2005.2758094912863, + "p95": 2109.8852323397296, + "p99": 2311.926480066613 + }, + "ttft_ms": { + "mean": 186.3455850543687, + "p50": 185.86589398910291, + "p90": 187.00031042099, + "p95": 190.6327125208918, + "p99": 190.94951929640956 + }, + "tpot_ms": { + "mean": 6.804822790357463, + "p50": 6.804042458734688, + "p90": 6.808869090739031, + "p95": 6.808880090071507, + "p99": 6.808956474964729 + }, + "itl_ms": { + "mean": 6.768538947538339, + "p50": 6.768758485865705, + "p90": 6.774978424220126, + "p95": 6.7753309364684435, + "p99": 6.777752989924467 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_1024.jsonl" + }, + { + "name": "c1_i4096_o128", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 28.907588530972134, + "request_throughput": 0.6918598546735099, + "input_token_throughput": 2833.8579647426964, + "output_token_throughput": 88.55806139820926, + "total_token_throughput": 2922.4160261409056, + "total_input_tokens": 81920, + "total_output_tokens": 2560, + "e2e_ms": { + "mean": 1043.4276448970195, + "p50": 1043.2338594982866, + "p90": 1044.0504656289704, + "p95": 1045.3570936457254, + "p99": 1047.5830067344941 + }, + "ttft_ms": { + "mean": 185.8626321045449, + "p50": 185.3025640011765, + "p90": 186.93417839240283, + "p95": 190.37860605167225, + "p99": 191.25218041474 + }, + "tpot_ms": { + "mean": 6.752480415688777, + "p50": 6.75444893298788, + "p90": 6.757666104827003, + "p95": 6.75770077612856, + "p99": 6.7581919096566105 + }, + "itl_ms": { + "mean": 6.751900093215627, + "p50": 6.7538843147812315, + "p90": 6.757120837780141, + "p95": 6.7571753729501465, + "p99": 6.757645178622911 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_128.jsonl" + }, + { + "name": "c1_i4096_o2048", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 42.9377163950121, + "request_throughput": 0.4657909567431797, + "input_token_throughput": 1907.8797588200641, + "output_token_throughput": 96.51188623718683, + "total_token_throughput": 2004.3916450572508, + "total_input_tokens": 81920, + "total_output_tokens": 4144, + "e2e_ms": { + "mean": 1589.2524215974845, + "p50": 1502.2164215042721, + "p90": 2021.3368221768178, + "p95": 2097.8192493930696, + "p99": 2452.9939467116487 + }, + "ttft_ms": { + "mean": 186.76971404929645, + "p50": 185.6516499829013, + "p90": 191.40778516884893, + "p95": 193.30401239567436, + "p99": 193.7424169119913 + }, + "tpot_ms": { + "mean": 6.802433210348238, + "p50": 6.803917665020255, + "p90": 6.808802393434643, + "p95": 6.809472329870284, + "p99": 6.809742051459609 + }, + "itl_ms": { + "mean": 6.764327205703543, + "p50": 6.764426513011319, + "p90": 6.771729031383434, + "p95": 6.772979212100163, + "p99": 6.774778441521013 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_2048.jsonl" + }, + { + "name": "c1_i4096_o256", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 40.42855142900953, + "request_throughput": 0.4946998913656102, + "input_token_throughput": 2026.2907550335394, + "output_token_throughput": 93.64668943551001, + "total_token_throughput": 2119.9374444690493, + "total_input_tokens": 81920, + "total_output_tokens": 3786, + "e2e_ms": { + "mean": 1469.5116658549523, + "p50": 1475.5392480292358, + "p90": 1803.493093926227, + "p95": 1913.7630071578315, + "p99": 1913.8662486354588 + }, + "ttft_ms": { + "mean": 189.04635770013556, + "p50": 186.6729814792052, + "p90": 194.76591550628658, + "p95": 207.5901315372903, + "p99": 207.82862710417248 + }, + "tpot_ms": { + "mean": 6.801977039878567, + "p50": 6.8026447482994765, + "p90": 6.814127616370421, + "p95": 6.814986886583095, + "p99": 6.821198251176533 + }, + "itl_ms": { + "mean": 6.762485412692762, + "p50": 6.76437729764892, + "p90": 6.7702427236612, + "p95": 6.772713689966694, + "p99": 6.773259632054296 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_256.jsonl" + }, + { + "name": "c1_i4096_o4096", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 43.119878391968086, + "request_throughput": 0.4638232004783526, + "input_token_throughput": 1899.8198291593324, + "output_token_throughput": 96.28969641930601, + "total_token_throughput": 1996.1095255786383, + "total_input_tokens": 81920, + "total_output_tokens": 4152, + "e2e_ms": { + "mean": 1593.6312893027207, + "p50": 1498.6803935316857, + "p90": 1957.292243535631, + "p95": 2383.6137426755163, + "p99": 2729.9412693403424 + }, + "ttft_ms": { + "mean": 187.92551439837553, + "p50": 185.15497801126912, + "p90": 193.47157318843531, + "p95": 205.29109993367456, + "p99": 207.28749755653553 + }, + "tpot_ms": { + "mean": 6.804683431925253, + "p50": 6.80432283275318, + "p90": 6.811035045306169, + "p95": 6.81644032220783, + "p99": 6.821541470131347 + }, + "itl_ms": { + "mean": 6.766404400702351, + "p50": 6.765622936847113, + "p90": 6.777374639968912, + "p95": 6.779508261833983, + "p99": 6.780120575364657 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_4096.jsonl" + }, + { + "name": "c1_i4096_o512", + "config": { + "concurrency": 1, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 41.37047825503396, + "request_throughput": 0.48343651907302765, + "input_token_throughput": 1980.1559821231212, + "output_token_throughput": 89.1940377689736, + "total_token_throughput": 2069.3500198920947, + "total_input_tokens": 81920, + "total_output_tokens": 3690, + "e2e_ms": { + "mean": 1434.7972685995046, + "p50": 1391.0726374888327, + "p90": 1746.7399196000774, + "p95": 2102.1896994323474, + "p99": 2396.88567985082 + }, + "ttft_ms": { + "mean": 185.99671100382693, + "p50": 185.7341964787338, + "p90": 186.8257091147825, + "p95": 187.35215915658046, + "p99": 190.6645454192767 + }, + "tpot_ms": { + "mean": 6.806494091020944, + "p50": 6.805695358530728, + "p90": 6.810304838032316, + "p95": 6.8114466669081, + "p99": 6.819568781625365 + }, + "itl_ms": { + "mean": 6.762844362184613, + "p50": 6.763746261668691, + "p90": 6.7704880665235025, + "p95": 6.774374872311334, + "p99": 6.777713881624482 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_4096_512.jsonl" + }, + { + "name": "c1_i512_o256", + "config": { + "concurrency": 1, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 39.877981167985126, + "request_throughput": 0.5015299023225482, + "input_token_throughput": 256.7833099891447, + "output_token_throughput": 96.79527114825181, + "total_token_throughput": 353.57858113739655, + "total_input_tokens": 10240, + "total_output_tokens": 3860, + "e2e_ms": { + "mean": 1408.4405573026743, + "p50": 1415.9690845117439, + "p90": 1817.8090272762347, + "p95": 1822.845825710101, + "p99": 1871.2951827363577 + }, + "ttft_ms": { + "mean": 118.17889174853917, + "p50": 110.91879848390818, + "p90": 127.67036562436267, + "p95": 176.59998708986677, + "p99": 184.48103661765344 + }, + "tpot_ms": { + "mean": 6.723991161659472, + "p50": 6.725678223120474, + "p90": 6.743545096530171, + "p95": 6.747081159763434, + "p99": 6.747433186365403 + }, + "itl_ms": { + "mean": 6.68911840516161, + "p50": 6.689323386256417, + "p90": 6.692883938661057, + "p95": 6.694141149762835, + "p99": 6.694941754767891 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_1_512_256.jsonl" + }, + { + "name": "c2_i512_o256", + "config": { + "concurrency": 2, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 40 + }, + "metrics": { + "success": 40, + "failed": 0, + "duration_s": 41.57707812497392, + "request_throughput": 0.9620685676796844, + "input_token_throughput": 492.5791066519984, + "output_token_throughput": 196.0455223789277, + "total_token_throughput": 688.6246290309261, + "total_input_tokens": 20480, + "total_output_tokens": 8151, + "e2e_ms": { + "mean": 1706.6990505249123, + "p50": 1820.5079275066964, + "p90": 2103.1951376004145, + "p95": 2108.239625924034, + "p99": 2165.3108836821048 + }, + "ttft_ms": { + "mean": 173.5474826011341, + "p50": 182.37082049017772, + "p90": 193.2680650963448, + "p95": 211.08191983948888, + "p99": 357.8624585439684 + }, + "tpot_ms": { + "mean": 7.607090684451674, + "p50": 7.5798865189869336, + "p90": 8.037224630473764, + "p95": 8.163422641582617, + "p99": 8.396796006667133 + }, + "itl_ms": { + "mean": 7.574754799232421, + "p50": 7.555747871211839, + "p90": 7.987534884216931, + "p95": 8.11219771662063, + "p99": 8.331527962665678 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_2_512_256.jsonl" + }, + { + "name": "c32_i1024_o1024", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 101.71446370997, + "request_throughput": 6.2921238205109615, + "input_token_throughput": 6443.134792203225, + "output_token_throughput": 1317.5903909019344, + "total_token_throughput": 7760.725183105159, + "total_input_tokens": 655360, + "total_output_tokens": 134018, + "e2e_ms": { + "mean": 4796.091996566065, + "p50": 4791.799418482697, + "p90": 6029.482252488378, + "p95": 6445.2264576451835, + "p99": 7146.159083393286 + }, + "ttft_ms": { + "mean": 197.9250626429348, + "p50": 136.0024324967526, + "p90": 276.72291316557676, + "p95": 311.2744802434463, + "p99": 984.941809528973 + }, + "tpot_ms": { + "mean": 22.027192085094246, + "p50": 22.450487293471358, + "p90": 23.485227822657954, + "p95": 23.778608281911556, + "p99": 24.211396291506148 + }, + "itl_ms": { + "mean": 21.90555821158396, + "p50": 22.335648349118106, + "p90": 23.356514053533175, + "p95": 23.602322259542458, + "p99": 24.127080775726817 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_1024.jsonl" + }, + { + "name": "c32_i1024_o128", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 53.75119669100968, + "request_throughput": 11.90671165293414, + "input_token_throughput": 12192.47273260456, + "output_token_throughput": 1523.8730492059929, + "total_token_throughput": 13716.345781810553, + "total_input_tokens": 655360, + "total_output_tokens": 81910, + "e2e_ms": { + "mean": 2502.1750740123025, + "p50": 2498.9854795276187, + "p90": 2553.823436773382, + "p95": 2579.4392172567314, + "p99": 2641.3214873848483 + }, + "ttft_ms": { + "mean": 681.1004243829302, + "p50": 688.7256299960427, + "p90": 992.9460007755551, + "p95": 1046.6297078499338, + "p99": 1091.5270992362641 + }, + "tpot_ms": { + "mean": 14.341129032501167, + "p50": 14.19918624043472, + "p90": 16.83120139866268, + "p95": 16.97889331681929, + "p99": 18.599388786677757 + }, + "itl_ms": { + "mean": 14.34079579673639, + "p50": 14.198858626177682, + "p90": 16.830803370964897, + "p95": 16.97858552774781, + "p99": 18.59905340557544 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_128.jsonl" + }, + { + "name": "c32_i1024_o2048", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 101.75704183196649, + "request_throughput": 6.28949101190309, + "input_token_throughput": 6440.438796188764, + "output_token_throughput": 1316.7442526607363, + "total_token_throughput": 7757.183048849501, + "total_input_tokens": 655360, + "total_output_tokens": 133988, + "e2e_ms": { + "mean": 4767.735684630679, + "p50": 4778.584390500328, + "p90": 6232.032449706458, + "p95": 6630.186968302586, + "p99": 7109.5137719484055 + }, + "ttft_ms": { + "mean": 204.18510429781236, + "p50": 136.04777900036424, + "p90": 291.9994773750659, + "p95": 335.85453994164675, + "p99": 1145.74000693392 + }, + "tpot_ms": { + "mean": 21.89133994235568, + "p50": 22.341144466274958, + "p90": 23.580969392419064, + "p95": 24.015448060274583, + "p99": 24.526134655402384 + }, + "itl_ms": { + "mean": 21.78653640287656, + "p50": 22.26405249724315, + "p90": 23.446381741910955, + "p95": 23.931203123018353, + "p99": 24.437188434985174 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_2048.jsonl" + }, + { + "name": "c32_i1024_o256", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 99.5990810120129, + "request_throughput": 6.425762100383316, + "input_token_throughput": 6579.980390792515, + "output_token_throughput": 1309.1385851670002, + "total_token_throughput": 7889.1189759595145, + "total_input_tokens": 655360, + "total_output_tokens": 130389, + "e2e_ms": { + "mean": 4694.1599163295905, + "p50": 4782.689154468244, + "p90": 5835.8982786070555, + "p95": 6056.401502765948, + "p99": 6274.461112323916 + }, + "ttft_ms": { + "mean": 200.9333179159512, + "p50": 144.9765920115169, + "p90": 280.0194281153381, + "p95": 331.4997089968528, + "p99": 921.881116280565 + }, + "tpot_ms": { + "mean": 22.148425063318644, + "p50": 22.54947068781904, + "p90": 24.063857943282756, + "p95": 24.504361418031394, + "p99": 25.32139924061117 + }, + "itl_ms": { + "mean": 22.04392262393563, + "p50": 22.467127987028874, + "p90": 23.94794534189806, + "p95": 24.434335787560258, + "p99": 25.1582555335446 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_256.jsonl" + }, + { + "name": "c32_i1024_o4096", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.10473308997462, + "request_throughput": 6.207280508078153, + "input_token_throughput": 6356.2552402720285, + "output_token_throughput": 1309.6294995707576, + "total_token_throughput": 7665.884739842786, + "total_input_tokens": 655360, + "total_output_tokens": 135029, + "e2e_ms": { + "mean": 4833.693425038291, + "p50": 4832.819982984802, + "p90": 6069.454644899815, + "p95": 6458.128238443168, + "p99": 7314.038717231598 + }, + "ttft_ms": { + "mean": 205.96744189742822, + "p50": 137.8191965050064, + "p90": 290.34566325717606, + "p95": 316.60519163415296, + "p99": 1157.5180955050746 + }, + "tpot_ms": { + "mean": 22.039607651379058, + "p50": 22.46821651881679, + "p90": 23.838373629966284, + "p95": 24.052396601548065, + "p99": 24.788685409547828 + }, + "itl_ms": { + "mean": 21.923150565519745, + "p50": 22.34886148876248, + "p90": 23.690222462612827, + "p95": 23.963653610176685, + "p99": 24.723726183834795 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_4096.jsonl" + }, + { + "name": "c32_i1024_o512", + "config": { + "concurrency": 32, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.0573151690187, + "request_throughput": 6.210136553143955, + "input_token_throughput": 6359.17983041941, + "output_token_throughput": 1304.3227429275164, + "total_token_throughput": 7663.5025733469265, + "total_input_tokens": 655360, + "total_output_tokens": 134420, + "e2e_ms": { + "mean": 4872.208154099917, + "p50": 4897.681755479425, + "p90": 6203.454227762995, + "p95": 6788.103030444472, + "p99": 7689.830216025585 + }, + "ttft_ms": { + "mean": 196.6105440258616, + "p50": 136.3337320217397, + "p90": 230.8978517074139, + "p95": 340.27282420720434, + "p99": 945.1565118343569 + }, + "tpot_ms": { + "mean": 22.382681351621876, + "p50": 22.651396316565084, + "p90": 24.314505681157875, + "p95": 25.381590820361843, + "p99": 27.967389366212952 + }, + "itl_ms": { + "mean": 22.267221438867587, + "p50": 22.502299660246532, + "p90": 24.191166484533962, + "p95": 25.25347770783478, + "p99": 27.81846139641072 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_1024_512.jsonl" + }, + { + "name": "c32_i16384_o1024", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 398.19623553403653, + "request_throughput": 1.6072477409076231, + "input_token_throughput": 26333.146987030497, + "output_token_throughput": 402.84660096011504, + "total_token_throughput": 26735.99358799061, + "total_input_tokens": 10485760, + "total_output_tokens": 160412, + "e2e_ms": { + "mean": 19184.298590931394, + "p50": 16635.307037999155, + "p90": 30508.484355715336, + "p95": 36508.48627480736, + "p99": 54198.776468519965 + }, + "ttft_ms": { + "mean": 1772.2575316559414, + "p50": 1323.4945705044083, + "p90": 2576.021732011577, + "p95": 3387.5279530038806, + "p99": 13518.463227089967 + }, + "tpot_ms": { + "mean": 70.46074460640907, + "p50": 71.73240135433585, + "p90": 82.03219558137306, + "p95": 85.49271957200986, + "p99": 101.8386985072882 + }, + "itl_ms": { + "mean": 70.11671918436993, + "p50": 71.3985684091518, + "p90": 81.70230217048596, + "p95": 85.39689406407119, + "p99": 100.46498175445457 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_1024.jsonl" + }, + { + "name": "c32_i16384_o128", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 359.2006303339731, + "request_throughput": 1.781734067128303, + "input_token_throughput": 29191.930955830117, + "output_token_throughput": 227.5497120481234, + "total_token_throughput": 29419.48066787824, + "total_input_tokens": 10485760, + "total_output_tokens": 81736, + "e2e_ms": { + "mean": 17518.50459199386, + "p50": 17511.15690698498, + "p90": 18241.288167028688, + "p95": 19731.453384371707, + "p99": 27923.40092844214 + }, + "ttft_ms": { + "mean": 3438.601437185207, + "p50": 3365.2316914813127, + "p90": 3878.172927122796, + "p95": 3983.9240162749716, + "p99": 13516.256364046138 + }, + "tpot_ms": { + "mean": 111.1261431259428, + "p50": 111.30386603538747, + "p90": 126.74013408946834, + "p95": 127.94849833767834, + "p99": 130.66239218253233 + }, + "itl_ms": { + "mean": 111.11157190744719, + "p50": 111.30349909062645, + "p90": 126.73962503148236, + "p95": 127.94812049980955, + "p99": 130.66199353753544 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_128.jsonl" + }, + { + "name": "c32_i16384_o2048", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 398.6252235029824, + "request_throughput": 1.605518071274814, + "input_token_throughput": 26304.808079766553, + "output_token_throughput": 403.5168637510881, + "total_token_throughput": 26708.32494351764, + "total_input_tokens": 10485760, + "total_output_tokens": 160852, + "e2e_ms": { + "mean": 19175.977931623493, + "p50": 16826.667610002914, + "p90": 31094.610782997916, + "p95": 37536.35124927386, + "p99": 50504.81318892155 + }, + "ttft_ms": { + "mean": 1715.0379632086697, + "p50": 1321.1069589888211, + "p90": 2372.458817012375, + "p95": 2963.249287707724, + "p99": 13514.417622748886 + }, + "tpot_ms": { + "mean": 70.47087580375502, + "p50": 71.53828355218127, + "p90": 81.78005991342835, + "p95": 86.6475722402378, + "p99": 95.97500238836932 + }, + "itl_ms": { + "mean": 70.14611375684596, + "p50": 71.22865627701796, + "p90": 81.27930098275836, + "p95": 86.08798771959307, + "p99": 95.6376215260082 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": false, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_2048.jsonl" + }, + { + "name": "c32_i16384_o256", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 383.92433379095746, + "request_throughput": 1.6669951437578658, + "input_token_throughput": 27312.048435328874, + "output_token_throughput": 348.16235449347874, + "total_token_throughput": 27660.21078982235, + "total_input_tokens": 10485760, + "total_output_tokens": 133668, + "e2e_ms": { + "mean": 18646.62216086317, + "p50": 18877.446957485517, + "p90": 23596.388958598254, + "p95": 24405.538778216567, + "p99": 29695.085078190335 + }, + "ttft_ms": { + "mean": 1826.8149450027522, + "p50": 1393.023353011813, + "p90": 2421.4535025763325, + "p95": 3388.6907908861745, + "p99": 13517.466220164213 + }, + "tpot_ms": { + "mean": 80.78933965516552, + "p50": 82.40988242028433, + "p90": 90.68880433252747, + "p95": 92.926508280831, + "p99": 101.6337257692619 + }, + "itl_ms": { + "mean": 80.45402753619312, + "p50": 82.00862929269339, + "p90": 90.5916024319385, + "p95": 92.75806571797227, + "p99": 101.5516533698931 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_256.jsonl" + }, + { + "name": "c32_i16384_o512", + "config": { + "concurrency": 32, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 393.371598202968, + "request_throughput": 1.6269603675600877, + "input_token_throughput": 26656.118662104476, + "output_token_throughput": 383.78215582840454, + "total_token_throughput": 27039.90081793288, + "total_input_tokens": 10485760, + "total_output_tokens": 150969, + "e2e_ms": { + "mean": 19000.893150751017, + "p50": 16956.22364000883, + "p90": 30767.245342204118, + "p95": 36338.548749694, + "p99": 41160.94916441828 + }, + "ttft_ms": { + "mean": 1747.0441726392892, + "p50": 1320.0604559970088, + "p90": 2374.9253470858093, + "p95": 3400.819807735387, + "p99": 13513.480063019677 + }, + "tpot_ms": { + "mean": 73.78208413979608, + "p50": 74.50801520232787, + "p90": 85.94058834740318, + "p95": 90.2959977150635, + "p99": 96.95683291620503 + }, + "itl_ms": { + "mean": 73.42788346346373, + "p50": 74.34369396839529, + "p90": 85.69958084748383, + "p95": 89.27350224578593, + "p99": 96.65062597269109 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_16384_512.jsonl" + }, + { + "name": "c32_i2048_o1024", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 112.12105303100543, + "request_throughput": 5.708116207426427, + "input_token_throughput": 11690.221992809322, + "output_token_throughput": 1270.4482891416405, + "total_token_throughput": 12960.670281950963, + "total_input_tokens": 1310720, + "total_output_tokens": 142444, + "e2e_ms": { + "mean": 5165.109592623776, + "p50": 4740.339960466372, + "p90": 7289.129797718488, + "p95": 8407.833625149215, + "p99": 11262.115388974782 + }, + "ttft_ms": { + "mean": 246.5300903953903, + "p50": 154.3558800185565, + "p90": 310.4189208941534, + "p95": 373.10424581228256, + "p99": 1896.2616414064541 + }, + "tpot_ms": { + "mean": 22.207007231194755, + "p50": 22.51868323658376, + "p90": 23.79985572262988, + "p95": 24.254527229962846, + "p99": 25.251460401786787 + }, + "itl_ms": { + "mean": 22.094839150458522, + "p50": 22.41546275374803, + "p90": 23.680054680674566, + "p95": 24.062508614105823, + "p99": 25.073584625372547 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_1024.jsonl" + }, + { + "name": "c32_i2048_o128", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 73.91267868096475, + "request_throughput": 8.658866265184132, + "input_token_throughput": 17733.358111097103, + "output_token_throughput": 1105.4125145790692, + "total_token_throughput": 18838.77062567617, + "total_input_tokens": 1310720, + "total_output_tokens": 81704, + "e2e_ms": { + "mean": 3492.798290261726, + "p50": 3497.8387630253565, + "p90": 3721.036367805209, + "p95": 3746.523601966328, + "p99": 4348.107912972922 + }, + "ttft_ms": { + "mean": 904.6822240371512, + "p50": 975.327655498404, + "p90": 1452.6418834982906, + "p95": 1464.6129804546945, + "p99": 1844.7583691641953 + }, + "tpot_ms": { + "mean": 20.425485261385226, + "p50": 19.787422429007574, + "p90": 25.10513298210732, + "p95": 26.381000489116587, + "p99": 27.057591470652333 + }, + "itl_ms": { + "mean": 20.42322585400549, + "p50": 19.787048114102525, + "p90": 25.104814078878817, + "p95": 26.38060599730504, + "p99": 27.057244394508036 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_128.jsonl" + }, + { + "name": "c32_i2048_o2048", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 111.78773300204193, + "request_throughput": 5.725136227499217, + "input_token_throughput": 11725.078993918396, + "output_token_throughput": 1282.421569434469, + "total_token_throughput": 13007.500563352864, + "total_input_tokens": 1310720, + "total_output_tokens": 143359, + "e2e_ms": { + "mean": 5185.733147027804, + "p50": 4744.378389994381, + "p90": 7545.672788115918, + "p95": 8810.36796895496, + "p99": 10858.06654340995 + }, + "ttft_ms": { + "mean": 246.50183836502038, + "p50": 150.90322599280626, + "p90": 331.4385375881102, + "p95": 442.18197880545597, + "p99": 1847.0357541838925 + }, + "tpot_ms": { + "mean": 22.172976606138974, + "p50": 22.429171546363605, + "p90": 23.695601993977686, + "p95": 24.280749206855816, + "p99": 25.62301267945426 + }, + "itl_ms": { + "mean": 22.066964132193903, + "p50": 22.299661700202833, + "p90": 23.59680629773218, + "p95": 24.194506006747183, + "p99": 25.525007229595307 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_2048.jsonl" + }, + { + "name": "c32_i2048_o256", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 103.61332845099969, + "request_throughput": 6.176811512262785, + "input_token_throughput": 12650.109977114183, + "output_token_throughput": 1258.1103411000622, + "total_token_throughput": 13908.220318214246, + "total_input_tokens": 1310720, + "total_output_tokens": 130357, + "e2e_ms": { + "mean": 4859.842472118635, + "p50": 4828.371057519689, + "p90": 6167.06687019323, + "p95": 6298.454219402629, + "p99": 6687.988225959125 + }, + "ttft_ms": { + "mean": 240.9834422237509, + "p50": 186.60073648788966, + "p90": 304.38226247788407, + "p95": 367.88477115624113, + "p99": 1848.7024522264287 + }, + "tpot_ms": { + "mean": 22.795846379865804, + "p50": 23.15601111415637, + "p90": 24.32344850623629, + "p95": 24.73872366202562, + "p99": 25.469112502702256 + }, + "itl_ms": { + "mean": 22.696593278349937, + "p50": 23.06816904172357, + "p90": 24.253067279885506, + "p95": 24.58853132190758, + "p99": 25.373207823111766 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_256.jsonl" + }, + { + "name": "c32_i2048_o4096", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 106.83542463398771, + "request_throughput": 5.990522358970396, + "input_token_throughput": 12268.589791171371, + "output_token_throughput": 1282.1496284521966, + "total_token_throughput": 13550.739419623567, + "total_input_tokens": 1310720, + "total_output_tokens": 136979, + "e2e_ms": { + "mean": 5036.072500929913, + "p50": 4737.078030011617, + "p90": 6963.614087650785, + "p95": 8060.132782527941, + "p99": 10159.852220952163 + }, + "ttft_ms": { + "mean": 250.26007797550847, + "p50": 189.39865499851294, + "p90": 331.10389779903926, + "p95": 439.4650830334279, + "p99": 1837.7938479348118 + }, + "tpot_ms": { + "mean": 22.45339401983756, + "p50": 22.80440429761633, + "p90": 24.29076330914555, + "p95": 24.64607409221561, + "p99": 25.843275491954916 + }, + "itl_ms": { + "mean": 22.335401602134166, + "p50": 22.700434707873512, + "p90": 24.13671753592239, + "p95": 24.522132348434965, + "p99": 25.597794042088253 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_4096.jsonl" + }, + { + "name": "c32_i2048_o512", + "config": { + "concurrency": 32, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 111.05525721795857, + "request_throughput": 5.7628969220604045, + "input_token_throughput": 11802.412896379708, + "output_token_throughput": 1281.6502664133527, + "total_token_throughput": 13084.063162793062, + "total_input_tokens": 1310720, + "total_output_tokens": 142334, + "e2e_ms": { + "mean": 5136.635271709019, + "p50": 4666.2061315146275, + "p90": 7534.419533464826, + "p95": 8816.123298896124, + "p99": 11310.105777755381 + }, + "ttft_ms": { + "mean": 242.25743507849984, + "p50": 148.94087848369963, + "p90": 326.93816607352346, + "p95": 437.2403310961068, + "p99": 1893.1734945153585 + }, + "tpot_ms": { + "mean": 22.15493830038853, + "p50": 22.43389434124179, + "p90": 23.98985761731914, + "p95": 24.51438626808776, + "p99": 25.546152305927095 + }, + "itl_ms": { + "mean": 22.048586154855595, + "p50": 22.311854000107793, + "p90": 23.930691744822965, + "p95": 24.459427310307202, + "p99": 25.435365217090617 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_2048_512.jsonl" + }, + { + "name": "c32_i4096_o1024", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 141.80376432603225, + "request_throughput": 4.513279340938547, + "input_token_throughput": 18486.39218048429, + "output_token_throughput": 947.3161776660905, + "total_token_throughput": 19433.708358150383, + "total_input_tokens": 2621440, + "total_output_tokens": 134333, + "e2e_ms": { + "mean": 6702.716318087732, + "p50": 6253.930084494641, + "p90": 9191.221753915306, + "p95": 11605.60482868459, + "p99": 14590.752592555365 + }, + "ttft_ms": { + "mean": 432.84981709230124, + "p50": 304.1287414962426, + "p90": 594.3656270916108, + "p95": 813.4859444078755, + "p99": 3483.0619139934424 + }, + "tpot_ms": { + "mean": 30.04166296734717, + "p50": 30.51881899689274, + "p90": 33.11392199115068, + "p95": 33.92001727738661, + "p99": 36.32087159977741 + }, + "itl_ms": { + "mean": 29.864745135826638, + "p50": 30.36130819243728, + "p90": 32.89914557815899, + "p95": 33.71280012456758, + "p99": 36.25344940904338 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_1024.jsonl" + }, + { + "name": "c32_i4096_o128", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 114.34404653601814, + "request_throughput": 5.597143177877664, + "input_token_throughput": 22925.89845658691, + "output_token_throughput": 710.9771121699102, + "total_token_throughput": 23636.87556875682, + "total_input_tokens": 2621440, + "total_output_tokens": 81296, + "e2e_ms": { + "mean": 5478.886110145686, + "p50": 5515.89411549503, + "p90": 5904.106940201018, + "p95": 6106.497004799892, + "p99": 7458.033464258187 + }, + "ttft_ms": { + "mean": 1053.8112139072837, + "p50": 996.5255100105423, + "p90": 1746.4399764954578, + "p95": 1909.4978857465321, + "p99": 3485.2048156253295 + }, + "tpot_ms": { + "mean": 35.09868246136891, + "p50": 35.47592700402231, + "p90": 41.460134331586104, + "p95": 41.98311302248539, + "p99": 43.261149156257495 + }, + "itl_ms": { + "mean": 35.06339184305249, + "p50": 35.37396917313953, + "p90": 41.45976171093648, + "p95": 41.9827239108721, + "p99": 43.204316071992984 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_128.jsonl" + }, + { + "name": "c32_i4096_o2048", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 144.45277938101208, + "request_throughput": 4.430513575041162, + "input_token_throughput": 18147.3836033686, + "output_token_throughput": 939.6703932014643, + "total_token_throughput": 19087.053996570063, + "total_input_tokens": 2621440, + "total_output_tokens": 135738, + "e2e_ms": { + "mean": 6700.040520485527, + "p50": 6357.520977006061, + "p90": 9260.33614158514, + "p95": 10714.146034815345, + "p99": 14733.258541694968 + }, + "ttft_ms": { + "mean": 434.39145126149015, + "p50": 309.5579859800637, + "p90": 571.2298724334687, + "p95": 747.313593828585, + "p99": 3483.196152728633 + }, + "tpot_ms": { + "mean": 29.876116420692682, + "p50": 30.3379228851805, + "p90": 32.78072649504329, + "p95": 33.53012496942364, + "p99": 35.93158563458935 + }, + "itl_ms": { + "mean": 29.728421008128368, + "p50": 30.180660484182958, + "p90": 32.61751984353158, + "p95": 33.27038472473239, + "p99": 35.61634739842644 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_2048.jsonl" + }, + { + "name": "c32_i4096_o256", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 137.277509423031, + "request_throughput": 4.662089243095107, + "input_token_throughput": 19095.917539717557, + "output_token_throughput": 921.5056459844019, + "total_token_throughput": 20017.42318570196, + "total_input_tokens": 2621440, + "total_output_tokens": 126502, + "e2e_ms": { + "mean": 6528.498834509901, + "p50": 6445.90070299455, + "p90": 8382.710035092896, + "p95": 8638.599968180642, + "p99": 10034.495897529998 + }, + "ttft_ms": { + "mean": 440.9964826984833, + "p50": 307.16633147676475, + "p90": 597.6938761305064, + "p95": 731.7515331407769, + "p99": 3478.6799135484034 + }, + "tpot_ms": { + "mean": 30.982427560307695, + "p50": 31.59422003819917, + "p90": 33.60870284218423, + "p95": 34.415974134281406, + "p99": 36.57922350277463 + }, + "itl_ms": { + "mean": 30.833046346702986, + "p50": 31.449339798004964, + "p90": 33.47908859361284, + "p95": 34.3315886835987, + "p99": 36.18148070887706 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_256.jsonl" + }, + { + "name": "c32_i4096_o4096", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 143.00942956196377, + "request_throughput": 4.475229374456724, + "input_token_throughput": 18330.53951777474, + "output_token_throughput": 961.5030311020271, + "total_token_throughput": 19292.04254887677, + "total_input_tokens": 2621440, + "total_output_tokens": 137504, + "e2e_ms": { + "mean": 6743.487641989396, + "p50": 6318.3071019884665, + "p90": 9436.950038914803, + "p95": 10500.857756665206, + "p99": 14281.367835595158 + }, + "ttft_ms": { + "mean": 429.60839072684394, + "p50": 308.00414050463587, + "p90": 571.643524611136, + "p95": 749.1604033275432, + "p99": 3469.335957507137 + }, + "tpot_ms": { + "mean": 29.60218774118301, + "p50": 30.001832055703364, + "p90": 32.42566294262988, + "p95": 33.17141477764495, + "p99": 35.01187529090304 + }, + "itl_ms": { + "mean": 29.463924546216248, + "p50": 29.88822948908127, + "p90": 32.286102683108695, + "p95": 33.004748232670295, + "p99": 34.77493098066052 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_4096.jsonl" + }, + { + "name": "c32_i4096_o512", + "config": { + "concurrency": 32, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 143.19192939798813, + "request_throughput": 4.46952564080048, + "input_token_throughput": 18307.177024718767, + "output_token_throughput": 929.291200694371, + "total_token_throughput": 19236.468225413137, + "total_input_tokens": 2621440, + "total_output_tokens": 133067, + "e2e_ms": { + "mean": 6735.349410550407, + "p50": 6316.515762504423, + "p90": 9159.898966323817, + "p95": 10401.616307868968, + "p99": 14892.06140367722 + }, + "ttft_ms": { + "mean": 430.74002514895255, + "p50": 310.50263551878743, + "p90": 543.8166258158163, + "p95": 691.6757039667568, + "p99": 3532.9511326598004 + }, + "tpot_ms": { + "mean": 30.56646949473112, + "p50": 30.9312377398962, + "p90": 33.449419883316686, + "p95": 34.146055170403926, + "p99": 36.9598451346281 + }, + "itl_ms": { + "mean": 30.39183315845209, + "p50": 30.743593022214746, + "p90": 33.260713398804185, + "p95": 33.86867077639849, + "p99": 36.898599968271185 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_4096_512.jsonl" + }, + { + "name": "c32_i512_o256", + "config": { + "concurrency": 32, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 640 + }, + "metrics": { + "success": 640, + "failed": 0, + "duration_s": 98.72821586497594, + "request_throughput": 6.482442677534918, + "input_token_throughput": 3319.010650897878, + "output_token_throughput": 1330.8150952478677, + "total_token_throughput": 4649.825746145746, + "total_input_tokens": 327680, + "total_output_tokens": 131389, + "e2e_ms": { + "mean": 4649.840651139948, + "p50": 4760.1729684975, + "p90": 5945.521820668364, + "p95": 6039.506064294255, + "p99": 6163.958358393284 + }, + "ttft_ms": { + "mean": 186.9829640457283, + "p50": 136.8874144682195, + "p90": 252.83180618425908, + "p95": 378.34352593054064, + "p99": 678.2834628055571 + }, + "tpot_ms": { + "mean": 21.818303136745314, + "p50": 22.270404707809327, + "p90": 23.48358913604192, + "p95": 23.81546439573604, + "p99": 24.53757630767593 + }, + "itl_ms": { + "mean": 21.733008667438355, + "p50": 22.199319917640157, + "p90": 23.35130398678313, + "p95": 23.740902679644243, + "p99": 24.421781921266398 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_32_512_256.jsonl" + }, + { + "name": "c4_i512_o256", + "config": { + "concurrency": 4, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 80 + }, + "metrics": { + "success": 80, + "failed": 0, + "duration_s": 46.0010106950067, + "request_throughput": 1.7390922240907156, + "input_token_throughput": 890.4152187344464, + "output_token_throughput": 360.4703407484031, + "total_token_throughput": 1250.8855594828497, + "total_input_tokens": 40960, + "total_output_tokens": 16582, + "e2e_ms": { + "mean": 1997.4373223354633, + "p50": 2098.9274029852822, + "p90": 2478.9669486810453, + "p95": 2523.6370165017433, + "p99": 2675.4348404653138 + }, + "ttft_ms": { + "mean": 164.63600002534804, + "p50": 118.1012575107161, + "p90": 209.9170045054044, + "p95": 347.47324945928995, + "p99": 374.55094301200006 + }, + "tpot_ms": { + "mean": 8.916417548236618, + "p50": 8.663080307750908, + "p90": 10.1927724996491, + "p95": 10.51017343682637, + "p99": 11.091826626165526 + }, + "itl_ms": { + "mean": 8.863431249984234, + "p50": 8.662887984275013, + "p90": 10.07585571055859, + "p95": 10.268174330863944, + "p99": 10.95105848389873 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_4_512_256.jsonl" + }, + { + "name": "c64_i512_o256", + "config": { + "concurrency": 64, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 1280 + }, + "metrics": { + "success": 1280, + "failed": 0, + "duration_s": 140.30180359998485, + "request_throughput": 9.12318991742554, + "input_token_throughput": 4671.0732377218765, + "output_token_throughput": 1901.52936850791, + "total_token_throughput": 6572.602606229786, + "total_input_tokens": 655360, + "total_output_tokens": 266788, + "e2e_ms": { + "mean": 6712.706156762669, + "p50": 6909.8945554869715, + "p90": 8549.926256021718, + "p95": 8741.368123874418, + "p99": 9237.345241245348 + }, + "ttft_ms": { + "mean": 229.05817989149, + "p50": 194.47254750411958, + "p90": 306.9184340885839, + "p95": 453.78009574778844, + "p99": 1147.805790141574 + }, + "tpot_ms": { + "mean": 31.257536585577196, + "p50": 32.00854534300386, + "p90": 34.015025776029766, + "p95": 35.352318105270726, + "p99": 36.91540531818083 + }, + "itl_ms": { + "mean": 31.135016112704886, + "p50": 31.86686346661208, + "p90": 33.88631216092506, + "p95": 35.253558867598436, + "p99": 36.734858386072254 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_64_512_256.jsonl" + }, + { + "name": "c8_i1024_o1024", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.01378972904058, + "request_throughput": 2.962206518050997, + "input_token_throughput": 3033.299474484221, + "output_token_throughput": 621.7671481389042, + "total_token_throughput": 3655.066622623125, + "total_input_tokens": 163840, + "total_output_tokens": 33584, + "e2e_ms": { + "mean": 2395.4786735364905, + "p50": 2329.696138011059, + "p90": 2964.6093561896123, + "p95": 3214.61368437449, + "p99": 3506.79572086723 + }, + "ttft_ms": { + "mean": 144.65098456239502, + "p50": 123.20308148628101, + "p90": 199.4970897270832, + "p95": 217.99946820829152, + "p99": 393.9990232139826 + }, + "tpot_ms": { + "mean": 10.80428496965314, + "p50": 10.846479013653251, + "p90": 11.449212558667767, + "p95": 11.820453448058938, + "p99": 12.236586239159527 + }, + "itl_ms": { + "mean": 10.748665729836475, + "p50": 10.797932864638046, + "p90": 11.402054409661389, + "p95": 11.773988858873494, + "p99": 12.175650934477838 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_1024.jsonl" + }, + { + "name": "c8_i1024_o128", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 30.886076872993726, + "request_throughput": 5.180327713938358, + "input_token_throughput": 5304.655579072879, + "output_token_throughput": 662.822930998413, + "total_token_throughput": 5967.4785100712925, + "total_input_tokens": 163840, + "total_output_tokens": 20472, + "e2e_ms": { + "mean": 1360.7545400929666, + "p50": 1356.6849144699518, + "p90": 1372.686949709896, + "p95": 1376.9633801421153, + "p99": 1412.428418964264 + }, + "ttft_ms": { + "mean": 307.6379093559808, + "p50": 328.5264960140921, + "p90": 341.3232935068663, + "p95": 345.5637474049581, + "p99": 389.5727182831615 + }, + "tpot_ms": { + "mean": 8.295535728493961, + "p50": 8.095632850579316, + "p90": 9.738737157010656, + "p95": 9.771883254412444, + "p99": 9.797184904290209 + }, + "itl_ms": { + "mean": 8.295176800331998, + "p50": 8.095306578765102, + "p90": 9.73838070324094, + "p95": 9.771548070340769, + "p99": 9.796842210201957 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_128.jsonl" + }, + { + "name": "c8_i1024_o2048", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 53.44451181899058, + "request_throughput": 2.993759219691231, + "input_token_throughput": 3065.6094409638204, + "output_token_throughput": 627.0802905545745, + "total_token_throughput": 3692.689731518395, + "total_input_tokens": 163840, + "total_output_tokens": 33514, + "e2e_ms": { + "mean": 2391.8724685270718, + "p50": 2429.516384989256, + "p90": 2975.693142216187, + "p95": 3190.5777254287395, + "p99": 3558.5536541033066 + }, + "ttft_ms": { + "mean": 147.13870290943305, + "p50": 123.122916003922, + "p90": 209.92188203381374, + "p95": 284.30513975326926, + "p99": 375.40317902341485 + }, + "tpot_ms": { + "mean": 10.762963745519746, + "p50": 10.834037995907572, + "p90": 11.480905401984911, + "p95": 11.595140581413345, + "p99": 12.305920331146154 + }, + "itl_ms": { + "mean": 10.704678773876426, + "p50": 10.78597629218042, + "p90": 11.407551552441637, + "p95": 11.554190884636792, + "p99": 12.25965226868399 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_2048.jsonl" + }, + { + "name": "c8_i1024_o256", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.3178577030194, + "request_throughput": 3.0582291979200438, + "input_token_throughput": 3131.626698670125, + "output_token_throughput": 620.5529321229509, + "total_token_throughput": 3752.179630793076, + "total_input_tokens": 163840, + "total_output_tokens": 32466, + "e2e_ms": { + "mean": 2345.425173519834, + "p50": 2381.8502814974636, + "p90": 2948.787523934152, + "p95": 2965.105928858975, + "p99": 3042.159925229498 + }, + "ttft_ms": { + "mean": 144.05868405883666, + "p50": 123.30061200191267, + "p90": 205.80871163983826, + "p95": 216.85114067804534, + "p99": 383.2814864401007 + }, + "tpot_ms": { + "mean": 10.888348926023786, + "p50": 10.947267107515419, + "p90": 11.52796358664845, + "p95": 11.752466147783078, + "p99": 12.285815981373782 + }, + "itl_ms": { + "mean": 10.835623968999094, + "p50": 10.886456865874756, + "p90": 11.473180943125312, + "p95": 11.660246153027902, + "p99": 12.233038689088968 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_256.jsonl" + }, + { + "name": "c8_i1024_o4096", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 55.321856701979414, + "request_throughput": 2.8921661263454164, + "input_token_throughput": 2961.5781133777064, + "output_token_throughput": 621.4903484750506, + "total_token_throughput": 3583.0684618527566, + "total_input_tokens": 163840, + "total_output_tokens": 34382, + "e2e_ms": { + "mean": 2438.4150670175586, + "p50": 2406.0954789747484, + "p90": 3173.2560309872497, + "p95": 3382.0623315201383, + "p99": 3756.3808398373653 + }, + "ttft_ms": { + "mean": 144.43321304788697, + "p50": 123.22676149779, + "p90": 207.3197172721848, + "p95": 220.03079180431183, + "p99": 380.43375951296184 + }, + "tpot_ms": { + "mean": 10.729683978884637, + "p50": 10.795890332354045, + "p90": 11.431676349473312, + "p95": 11.55948920758737, + "p99": 12.092195742100818 + }, + "itl_ms": { + "mean": 10.677337930646178, + "p50": 10.729629197808688, + "p90": 11.383656778432892, + "p95": 11.513900642614868, + "p99": 12.038068449618027 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_4096.jsonl" + }, + { + "name": "c8_i1024_o512", + "config": { + "concurrency": 8, + "input_len": 1024, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 54.03436242096359, + "request_throughput": 2.961078706795755, + "input_token_throughput": 3032.144595758853, + "output_token_throughput": 632.2088106428112, + "total_token_throughput": 3664.353406401664, + "total_input_tokens": 163840, + "total_output_tokens": 34161, + "e2e_ms": { + "mean": 2425.822611462354, + "p50": 2396.973063965561, + "p90": 3130.640160792973, + "p95": 3208.353921084199, + "p99": 3616.7711639672048 + }, + "ttft_ms": { + "mean": 144.40323555427312, + "p50": 123.08634052169509, + "p90": 203.37686730199493, + "p95": 216.0994939156808, + "p99": 375.8623972674832 + }, + "tpot_ms": { + "mean": 10.739382575641525, + "p50": 10.858331415132689, + "p90": 11.355513657332015, + "p95": 11.482381241675077, + "p99": 11.759983664804802 + }, + "itl_ms": { + "mean": 10.685416116247126, + "p50": 10.819440029603811, + "p90": 11.31777107405648, + "p95": 11.439431671480277, + "p99": 11.715367241071098 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_1024_512.jsonl" + }, + { + "name": "c8_i16384_o1024", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 138.9691290059709, + "request_throughput": 1.151334840654614, + "input_token_throughput": 18863.470029285196, + "output_token_throughput": 287.5890515100144, + "total_token_throughput": 19151.05908079521, + "total_input_tokens": 2621440, + "total_output_tokens": 39966, + "e2e_ms": { + "mean": 6298.755326521496, + "p50": 5593.749911990017, + "p90": 10127.106156485386, + "p95": 12516.83689956845, + "p99": 14588.597605068933 + }, + "ttft_ms": { + "mean": 967.7164456628816, + "p50": 890.4675914964173, + "p90": 1378.9012019173242, + "p95": 1600.4013906058397, + "p99": 3663.330937565768 + }, + "tpot_ms": { + "mean": 21.472376172379033, + "p50": 22.257800259102922, + "p90": 26.010914360549727, + "p95": 27.316078669897212, + "p99": 29.59252122410235 + }, + "itl_ms": { + "mean": 21.351223905498287, + "p50": 22.178765532877954, + "p90": 25.717352797940066, + "p95": 27.31455100890858, + "p99": 29.54132950093783 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_1024.jsonl" + }, + { + "name": "c8_i16384_o128", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 111.92959954298567, + "request_throughput": 1.4294699583781971, + "input_token_throughput": 23420.43579806838, + "output_token_throughput": 181.32826422027432, + "total_token_throughput": 23601.764062288657, + "total_input_tokens": 2621440, + "total_output_tokens": 20296, + "e2e_ms": { + "mean": 5167.535090905221, + "p50": 5161.67644050438, + "p90": 5232.32137238374, + "p95": 5417.745105508948, + "p99": 5933.665734616807 + }, + "ttft_ms": { + "mean": 2556.643709580385, + "p50": 2640.1203620189335, + "p90": 3866.4832159643993, + "p95": 4119.31119611545, + "p99": 4136.104622313287 + }, + "tpot_ms": { + "mean": 20.737635027964952, + "p50": 20.266374896487427, + "p90": 33.92593250609934, + "p95": 33.96777861103941, + "p99": 33.99695902494602 + }, + "itl_ms": { + "mean": 20.736728008977952, + "p50": 20.266098003799378, + "p90": 33.925570322973066, + "p95": 33.967433898780854, + "p99": 33.99658160026847 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_128.jsonl" + }, + { + "name": "c8_i16384_o2048", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 136.25688857299974, + "request_throughput": 1.1742525583525258, + "input_token_throughput": 19238.953916047783, + "output_token_throughput": 275.00994916616156, + "total_token_throughput": 19513.963865213947, + "total_input_tokens": 2621440, + "total_output_tokens": 37472, + "e2e_ms": { + "mean": 6163.9701214990055, + "p50": 5399.014000024181, + "p90": 9542.940336407628, + "p95": 11684.397922272918, + "p99": 14272.990592987044 + }, + "ttft_ms": { + "mean": 961.941650908193, + "p50": 889.2011054849718, + "p90": 1358.9895943936426, + "p95": 1572.8736859047788, + "p99": 3673.563928353831 + }, + "tpot_ms": { + "mean": 22.523256550346055, + "p50": 22.868919465467282, + "p90": 26.401673455046737, + "p95": 28.431650291592916, + "p99": 30.67564380545814 + }, + "itl_ms": { + "mean": 22.396629750278695, + "p50": 22.68798743846689, + "p90": 26.286321661482393, + "p95": 27.869237309002852, + "p99": 30.610476568286856 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_2048.jsonl" + }, + { + "name": "c8_i16384_o256", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 127.81865492399083, + "request_throughput": 1.2517734605730773, + "input_token_throughput": 20509.056378029298, + "output_token_throughput": 253.68753895339125, + "total_token_throughput": 20762.743916982687, + "total_input_tokens": 2621440, + "total_output_tokens": 32426, + "e2e_ms": { + "mean": 5890.637765062638, + "p50": 5999.108154996065, + "p90": 7295.3310133947525, + "p95": 7584.774667481543, + "p99": 7970.696000051683 + }, + "ttft_ms": { + "mean": 1027.092823043131, + "p50": 896.6540365072433, + "p90": 1433.758345700334, + "p95": 1938.521144268454, + "p99": 3668.7240775686205 + }, + "tpot_ms": { + "mean": 24.18784923644043, + "p50": 24.955616296678972, + "p90": 28.24579013451567, + "p95": 29.34601891272757, + "p99": 30.471414801470594 + }, + "itl_ms": { + "mean": 24.09058552564381, + "p50": 24.940828342397996, + "p90": 28.163052283118482, + "p95": 29.2733077929844, + "p99": 30.42853842177255 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_256.jsonl" + }, + { + "name": "c8_i16384_o512", + "config": { + "concurrency": 8, + "input_len": 16384, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 136.6128168310388, + "request_throughput": 1.1711931845888677, + "input_token_throughput": 19188.829136304008, + "output_token_throughput": 286.08589520804196, + "total_token_throughput": 19474.91503151205, + "total_input_tokens": 2621440, + "total_output_tokens": 39083, + "e2e_ms": { + "mean": 6250.800366418844, + "p50": 5613.646615500329, + "p90": 9697.346573503455, + "p95": 10966.727837751389, + "p99": 12572.724322149297 + }, + "ttft_ms": { + "mean": 969.4523646234302, + "p50": 872.605326003395, + "p90": 1416.2633731088135, + "p95": 1904.348115285393, + "p99": 3667.981622895457 + }, + "tpot_ms": { + "mean": 21.65413662900705, + "p50": 22.15968788136079, + "p90": 25.971828724872314, + "p95": 26.893490572898347, + "p99": 29.249297661201627 + }, + "itl_ms": { + "mean": 21.52535903197245, + "p50": 22.04750119635838, + "p90": 25.842861547491193, + "p95": 26.289499275910167, + "p99": 29.012356170540826 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_16384_512.jsonl" + }, + { + "name": "c8_i2048_o1024", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 59.21719591296278, + "request_throughput": 2.7019178725579547, + "input_token_throughput": 5533.527802998691, + "output_token_throughput": 606.8169802031097, + "total_token_throughput": 6140.344783201801, + "total_input_tokens": 327680, + "total_output_tokens": 35934, + "e2e_ms": { + "mean": 2572.1823010015214, + "p50": 2315.6313379877247, + "p90": 3573.3853886660645, + "p95": 4486.641788340058, + "p99": 5823.30393439799 + }, + "ttft_ms": { + "mean": 155.89313780001248, + "p50": 127.48752950574271, + "p90": 214.49003661982712, + "p95": 244.40497531904856, + "p99": 621.821830929839 + }, + "tpot_ms": { + "mean": 10.829963419578188, + "p50": 10.84090427110567, + "p90": 11.683323234763234, + "p95": 11.858814439893099, + "p99": 12.107412703150931 + }, + "itl_ms": { + "mean": 10.77232341137624, + "p50": 10.792881839767215, + "p90": 11.5925756148027, + "p95": 11.796772338880846, + "p99": 12.05002575920413 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_1024.jsonl" + }, + { + "name": "c8_i2048_o128", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 36.785884183016606, + "request_throughput": 4.349494474673228, + "input_token_throughput": 8907.76468413077, + "output_token_throughput": 554.3702551375695, + "total_token_throughput": 9462.134939268339, + "total_input_tokens": 327680, + "total_output_tokens": 20393, + "e2e_ms": { + "mean": 1636.6455332696205, + "p50": 1650.3047684964258, + "p90": 1672.4332634243183, + "p95": 1708.9987661631312, + "p99": 1726.4511093095643 + }, + "ttft_ms": { + "mean": 337.84846669113904, + "p50": 368.39110546861775, + "p90": 501.928889635019, + "p95": 507.02435676357754, + "p99": 613.4727610013215 + }, + "tpot_ms": { + "mean": 10.27273139283467, + "p50": 9.955073681139336, + "p90": 12.024120250524309, + "p95": 12.093548784417461, + "p99": 12.511676546304457 + }, + "itl_ms": { + "mean": 10.269848071243864, + "p50": 9.954793488251148, + "p90": 12.023784178144144, + "p95": 12.093203529985034, + "p99": 12.511328950411581 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_128.jsonl" + }, + { + "name": "c8_i2048_o2048", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.54973888397217, + "request_throughput": 2.780203752489321, + "input_token_throughput": 5693.857285098129, + "output_token_throughput": 626.970698733248, + "total_token_throughput": 6320.827983831377, + "total_input_tokens": 327680, + "total_output_tokens": 36082, + "e2e_ms": { + "mean": 2571.8697952848743, + "p50": 2376.0124715045094, + "p90": 3531.4382432959974, + "p95": 4101.499575420167, + "p99": 5201.484706590415 + }, + "ttft_ms": { + "mean": 156.05718818696914, + "p50": 127.60109599912539, + "p90": 201.8362101982347, + "p95": 275.62024475191726, + "p99": 603.0409268877702 + }, + "tpot_ms": { + "mean": 10.757719886560922, + "p50": 10.89242498692186, + "p90": 11.446393503580977, + "p95": 11.598768451039856, + "p99": 11.99307822632247 + }, + "itl_ms": { + "mean": 10.702844962158807, + "p50": 10.82017472881797, + "p90": 11.400685704016238, + "p95": 11.540790897681546, + "p99": 11.943829748109716 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_2048.jsonl" + }, + { + "name": "c8_i2048_o256", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.18541706399992, + "request_throughput": 3.0659906349656425, + "input_token_throughput": 6279.148820409636, + "output_token_throughput": 604.0001550882316, + "total_token_throughput": 6883.1489754978675, + "total_input_tokens": 327680, + "total_output_tokens": 31520, + "e2e_ms": { + "mean": 2334.2623964919767, + "p50": 2330.2118405117653, + "p90": 2971.9119044952095, + "p95": 3004.2558109009406, + "p99": 3124.591278433217 + }, + "ttft_ms": { + "mean": 158.73426251928322, + "p50": 127.42687200079672, + "p90": 208.59998430823907, + "p95": 268.0313628050499, + "p99": 611.5358663420193 + }, + "tpot_ms": { + "mean": 11.131267257626003, + "p50": 11.17333754727414, + "p90": 11.92786350601456, + "p95": 12.152475340421605, + "p99": 12.446548674304928 + }, + "itl_ms": { + "mean": 11.069680648653042, + "p50": 11.113051173516164, + "p90": 11.841425316666124, + "p95": 12.086030156396962, + "p99": 12.403057491909903 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_256.jsonl" + }, + { + "name": "c8_i2048_o4096", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 59.21520890400279, + "request_throughput": 2.702008537357105, + "input_token_throughput": 5533.713484507351, + "output_token_throughput": 614.8251551222507, + "total_token_throughput": 6148.538639629602, + "total_input_tokens": 327680, + "total_output_tokens": 36407, + "e2e_ms": { + "mean": 2624.6837296948797, + "p50": 2390.2249354869127, + "p90": 3673.1235857703714, + "p95": 4094.224581937306, + "p99": 5661.190954251214 + }, + "ttft_ms": { + "mean": 164.46339333379, + "p50": 130.26967650512233, + "p90": 212.50965080107562, + "p95": 304.53456362884083, + "p99": 685.4601935023675 + }, + "tpot_ms": { + "mean": 10.838255479339688, + "p50": 10.899910423136323, + "p90": 11.584270936940777, + "p95": 11.765093191234756, + "p99": 11.979969078357305 + }, + "itl_ms": { + "mean": 10.793764061875354, + "p50": 10.857900357343716, + "p90": 11.548314111231585, + "p95": 11.743018766553384, + "p99": 11.925272947301288 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_4096.jsonl" + }, + { + "name": "c8_i2048_o512", + "config": { + "concurrency": 8, + "input_len": 2048, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 57.762821699958295, + "request_throughput": 2.7699477845992333, + "input_token_throughput": 5672.85306285923, + "output_token_throughput": 605.804892665506, + "total_token_throughput": 6278.657955524736, + "total_input_tokens": 327680, + "total_output_tokens": 34993, + "e2e_ms": { + "mean": 2521.3216953932715, + "p50": 2340.7801749999635, + "p90": 3667.4506752635352, + "p95": 3808.3090486150477, + "p99": 4671.025379123165 + }, + "ttft_ms": { + "mean": 154.17553363658953, + "p50": 127.54814099753276, + "p90": 207.90915787802078, + "p95": 248.11484048841533, + "p99": 617.8795976977563 + }, + "tpot_ms": { + "mean": 10.860595376329208, + "p50": 10.945198745940369, + "p90": 11.707197147642002, + "p95": 11.808376366833382, + "p99": 12.001137625009958 + }, + "itl_ms": { + "mean": 10.805698258728144, + "p50": 10.888047823227195, + "p90": 11.661599113280404, + "p95": 11.776820527168061, + "p99": 11.951079170990065 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_2048_512.jsonl" + }, + { + "name": "c8_i4096_o1024", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 64.92397661705036, + "request_throughput": 2.464420824740128, + "input_token_throughput": 10094.267698135563, + "output_token_throughput": 518.1290757714572, + "total_token_throughput": 10612.39677390702, + "total_input_tokens": 655360, + "total_output_tokens": 33639, + "e2e_ms": { + "mean": 2846.8765726112906, + "p50": 2693.235291488236, + "p90": 3686.392815841827, + "p95": 4602.444140485022, + "p99": 5170.714695255737 + }, + "ttft_ms": { + "mean": 256.3253793414333, + "p50": 200.92808597837575, + "p90": 408.3847037283703, + "p95": 442.47977496997936, + "p99": 1016.1522281781067 + }, + "tpot_ms": { + "mean": 12.360228898781688, + "p50": 12.529275761887078, + "p90": 13.544903270056238, + "p95": 13.768095517160951, + "p99": 14.462114144815686 + }, + "itl_ms": { + "mean": 12.290908117324742, + "p50": 12.467834575160465, + "p90": 13.514858231321586, + "p95": 13.711070186527868, + "p99": 14.255995922067301 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_1024.jsonl" + }, + { + "name": "c8_i4096_o128", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 46.47085372102447, + "request_throughput": 3.4430183047748133, + "input_token_throughput": 14102.602976357635, + "output_token_throughput": 440.039258214626, + "total_token_throughput": 14542.642234572262, + "total_input_tokens": 655360, + "total_output_tokens": 20449, + "e2e_ms": { + "mean": 2091.876247622349, + "p50": 2089.7147939831484, + "p90": 2107.6002722315025, + "p95": 2145.5566821066895, + "p99": 2225.516522292164 + }, + "ttft_ms": { + "mean": 707.0257856092212, + "p50": 675.6646229769103, + "p90": 1041.339729988249, + "p95": 1047.2809197148308, + "p99": 1071.5035220043499 + }, + "tpot_ms": { + "mean": 10.92299427938467, + "p50": 11.188700704653858, + "p90": 14.991371856230554, + "p95": 15.027174705320485, + "p99": 15.062308338558756 + }, + "itl_ms": { + "mean": 10.921993762760962, + "p50": 11.188328901615623, + "p90": 14.9909882158624, + "p95": 15.026840265321464, + "p99": 15.061849427742215 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_128.jsonl" + }, + { + "name": "c8_i4096_o2048", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 2048, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 66.29023894300917, + "request_throughput": 2.413628349379683, + "input_token_throughput": 9886.221719059182, + "output_token_throughput": 513.3334943815076, + "total_token_throughput": 10399.555213440688, + "total_input_tokens": 655360, + "total_output_tokens": 34029, + "e2e_ms": { + "mean": 2865.3563139934704, + "p50": 2743.0460805189796, + "p90": 3883.7065048923246, + "p95": 4853.713436282124, + "p99": 6028.238608094397 + }, + "ttft_ms": { + "mean": 253.25430698540004, + "p50": 201.3168605044484, + "p90": 340.9121041302569, + "p95": 423.3174357010287, + "p99": 1022.5903978734273 + }, + "tpot_ms": { + "mean": 12.385675609766118, + "p50": 12.41990564159736, + "p90": 13.593211750988889, + "p95": 14.051944799220749, + "p99": 14.659802628899744 + }, + "itl_ms": { + "mean": 12.327093622960454, + "p50": 12.35750935978226, + "p90": 13.560235076824098, + "p95": 13.993455026303808, + "p99": 14.591190706431963 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_2048.jsonl" + }, + { + "name": "c8_i4096_o256", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 62.25946529000066, + "request_throughput": 2.5698903653401146, + "input_token_throughput": 10526.27093643311, + "output_token_throughput": 516.499777989013, + "total_token_throughput": 11042.770714422124, + "total_input_tokens": 655360, + "total_output_tokens": 32157, + "e2e_ms": { + "mean": 2789.2727036618453, + "p50": 2782.83986801398, + "p90": 3511.2314169178717, + "p95": 3625.363992372877, + "p99": 3833.623016313649 + }, + "ttft_ms": { + "mean": 256.85724185968866, + "p50": 200.55336799123324, + "p90": 421.0858193633612, + "p95": 457.12938645447144, + "p99": 1013.5523934691441 + }, + "tpot_ms": { + "mean": 12.6272613432991, + "p50": 12.802867527432571, + "p90": 13.773254597121301, + "p95": 14.045325650575263, + "p99": 14.568506100596926 + }, + "itl_ms": { + "mean": 12.558048593675748, + "p50": 12.695122376988785, + "p90": 13.6908571208121, + "p95": 13.99627949972378, + "p99": 14.511433540532495 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_256.jsonl" + }, + { + "name": "c8_i4096_o4096", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 4096, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 64.2815860339906, + "request_throughput": 2.4890487287509635, + "input_token_throughput": 10195.143592963947, + "output_token_throughput": 543.8415906775308, + "total_token_throughput": 10738.985183641476, + "total_input_tokens": 655360, + "total_output_tokens": 34959, + "e2e_ms": { + "mean": 2921.1263272722135, + "p50": 2646.676472533727, + "p90": 4069.956865196581, + "p95": 5570.157847894009, + "p99": 8112.523980666883 + }, + "ttft_ms": { + "mean": 248.18690962783876, + "p50": 199.7099299915135, + "p90": 340.966394089628, + "p95": 462.5693424663051, + "p99": 1014.7484800923844 + }, + "tpot_ms": { + "mean": 12.268584562652679, + "p50": 12.345150528310768, + "p90": 13.420375906472373, + "p95": 13.678864128672437, + "p99": 14.315362949122035 + }, + "itl_ms": { + "mean": 12.203152687226083, + "p50": 12.302508034345959, + "p90": 13.362345801675275, + "p95": 13.600341813471513, + "p99": 14.240583484598446 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_4096.jsonl" + }, + { + "name": "c8_i4096_o512", + "config": { + "concurrency": 8, + "input_len": 4096, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 65.25227290898329, + "request_throughput": 2.4520218663214224, + "input_token_throughput": 10043.481564452546, + "output_token_throughput": 538.7398543041455, + "total_token_throughput": 10582.221418756693, + "total_input_tokens": 655360, + "total_output_tokens": 35154, + "e2e_ms": { + "mean": 2931.060687192439, + "p50": 2709.595823456766, + "p90": 3999.5783280173773, + "p95": 5124.890309749753, + "p99": 6513.321161492495 + }, + "ttft_ms": { + "mean": 247.69492733612424, + "p50": 200.21844847360626, + "p90": 325.43213293538423, + "p95": 442.6875208271668, + "p99": 1015.5376534443344 + }, + "tpot_ms": { + "mean": 12.24628101497471, + "p50": 12.375387132825676, + "p90": 13.439804439564156, + "p95": 13.708735240033729, + "p99": 14.063654821676147 + }, + "itl_ms": { + "mean": 12.189888993120336, + "p50": 12.31609789090539, + "p90": 13.397326564884626, + "p95": 13.661249612179654, + "p99": 14.01123272006841 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_4096_512.jsonl" + }, + { + "name": "c8_i512_o256", + "config": { + "concurrency": 8, + "input_len": 512, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 52.72627985099098, + "request_throughput": 3.0345399002579705, + "input_token_throughput": 1553.684428932081, + "output_token_throughput": 629.4773705597628, + "total_token_throughput": 2183.161799491844, + "total_input_tokens": 81920, + "total_output_tokens": 33190, + "e2e_ms": { + "mean": 2369.8188398560887, + "p50": 2406.275746994652, + "p90": 2961.9523657194804, + "p95": 3039.6370284666773, + "p99": 3126.469637452392 + }, + "ttft_ms": { + "mean": 135.86247368766635, + "p50": 116.1904759646859, + "p90": 201.228951633675, + "p95": 271.8837070133306, + "p99": 289.0714289085008 + }, + "tpot_ms": { + "mean": 10.800091042884167, + "p50": 10.888139287493697, + "p90": 11.647774160941, + "p95": 11.890522297109356, + "p99": 12.06757466577572 + }, + "itl_ms": { + "mean": 10.746752771231769, + "p50": 10.832085453558161, + "p90": 11.590206155203498, + "p95": 11.81536591946112, + "p99": 12.02890745581324 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0711_8_512_256.jsonl" + }, + { + "name": "c128_i16384_o1024", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 1024, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1401.2432732849848, + "request_throughput": 1.8269490022232189, + "input_token_throughput": 29932.732452425218, + "output_token_throughput": 453.17041808974545, + "total_token_throughput": 30385.90287051496, + "total_input_tokens": 41943040, + "total_output_tokens": 635002, + "e2e_ms": { + "mean": 69085.30075752884, + "p50": 59839.19886650983, + "p90": 112047.84775544539, + "p95": 135447.23007164142, + "p99": 185844.62699736594 + }, + "ttft_ms": { + "mean": 6827.033814014499, + "p50": 5296.662732493132, + "p90": 8734.584317711415, + "p95": 13719.54655988375, + "p99": 53845.72860275387 + }, + "tpot_ms": { + "mean": 254.24761383279738, + "p50": 264.87820864011053, + "p90": 266.3930666301598, + "p95": 266.592708826959, + "p99": 267.0054887261078 + }, + "itl_ms": { + "mean": 253.07364340401926, + "p50": 263.45366783234203, + "p90": 264.834891925602, + "p95": 264.927611077207, + "p99": 265.1289296694187 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_128_16384_1024.jsonl" + }, + { + "name": "c128_i16384_o2048", + "config": { + "concurrency": 128, + "input_len": 16384, + "output_len": 2048, + "dataset": "random", + "num_prompts": 2560 + }, + "metrics": { + "success": 2560, + "failed": 0, + "duration_s": 1400.62394244998, + "request_throughput": 1.8277568463680782, + "input_token_throughput": 29945.968170894594, + "output_token_throughput": 468.4455121139211, + "total_token_throughput": 30414.413683008515, + "total_input_tokens": 41943040, + "total_output_tokens": 656116, + "e2e_ms": { + "mean": 69277.39991989332, + "p50": 59091.808564495295, + "p90": 114759.83570949172, + "p95": 141505.20173237895, + "p99": 189264.63393578478 + }, + "ttft_ms": { + "mean": 5874.331173899714, + "p50": 4223.068817984313, + "p90": 8200.802748126443, + "p95": 13475.141925734402, + "p99": 53934.113027165215 + }, + "tpot_ms": { + "mean": 249.3775254623817, + "p50": 254.3313727734818, + "p90": 266.0176135906007, + "p95": 266.260407924198, + "p99": 266.7113317080453 + }, + "itl_ms": { + "mean": 248.26982724391488, + "p50": 253.29337461046123, + "p90": 264.63717776394753, + "p95": 264.79745950692586, + "p99": 264.95209586103056 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_128_16384_2048.jsonl" + }, + { + "name": "c1_i131072_o128", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 182.19263550196774, + "request_throughput": 0.10977392112967153, + "input_token_throughput": 14388.287390308307, + "output_token_throughput": 12.72279745892893, + "total_token_throughput": 14401.010187767235, + "total_input_tokens": 2621440, + "total_output_tokens": 2318, + "e2e_ms": { + "mean": 6192.48906809953, + "p50": 6180.761422525393, + "p90": 6213.763176015345, + "p95": 6221.419949684059, + "p99": 6221.434263539268 + }, + "ttft_ms": { + "mean": 5392.411926749628, + "p50": 5379.820790520171, + "p90": 5415.663412271533, + "p95": 5419.101967004826, + "p99": 5420.543884577928 + }, + "tpot_ms": { + "mean": 6.963251589076258, + "p50": 6.964099764229672, + "p90": 6.971392656594688, + "p95": 6.973061491507511, + "p99": 6.9767391713204505 + }, + "itl_ms": { + "mean": 6.962635385825456, + "p50": 6.963509944296697, + "p90": 6.9707969511412635, + "p95": 6.972460199486586, + "p99": 6.976156700761867 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_128.jsonl" + }, + { + "name": "c1_i131072_o256", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 195.04452994198073, + "request_throughput": 0.10254068650861081, + "input_token_throughput": 13440.212862056636, + "output_token_throughput": 19.005916244371015, + "total_token_throughput": 13459.218778301007, + "total_input_tokens": 2621440, + "total_output_tokens": 3707, + "e2e_ms": { + "mean": 6686.716299253749, + "p50": 6740.412624028977, + "p90": 7094.8766680958215, + "p95": 7103.287010962958, + "p99": 7119.59390459524 + }, + "ttft_ms": { + "mean": 5397.539071351639, + "p50": 5400.779595016502, + "p90": 5420.498099375982, + "p95": 5431.664711123449, + "p99": 5440.220488613122 + }, + "tpot_ms": { + "mean": 6.998566515518891, + "p50": 7.004880447191741, + "p90": 7.022558806460007, + "p95": 7.030111121322866, + "p99": 7.030822165726561 + }, + "itl_ms": { + "mean": 6.937857768655009, + "p50": 6.934740034527012, + "p90": 6.970667251731075, + "p95": 6.977926703932042, + "p99": 6.977967237039455 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_256.jsonl" + }, + { + "name": "c1_i131072_o512", + "config": { + "concurrency": 1, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 201.48357103700982, + "request_throughput": 0.09926367642315745, + "input_token_throughput": 13010.688596136093, + "output_token_throughput": 20.185268600649067, + "total_token_throughput": 13030.87386473674, + "total_input_tokens": 2621440, + "total_output_tokens": 4067, + "e2e_ms": { + "mean": 6917.335643596016, + "p50": 6881.105848500738, + "p90": 7239.1232147521805, + "p95": 7364.33622519835, + "p99": 8109.9643394339355 + }, + "ttft_ms": { + "mean": 5500.612655151053, + "p50": 5505.672536994098, + "p90": 5544.737549085403, + "p95": 5555.720844605821, + "p99": 5562.254832139588 + }, + "tpot_ms": { + "mean": 7.003788900084021, + "p50": 7.004890883533367, + "p90": 7.0144188913927445, + "p95": 7.020465257247111, + "p99": 7.030904896468426 + }, + "itl_ms": { + "mean": 6.931159709011044, + "p50": 6.933711249570328, + "p90": 6.942321813002504, + "p95": 6.943215882998016, + "p99": 6.953637245724014 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_131072_512.jsonl" + }, + { + "name": "c1_i262144_o256", + "config": { + "concurrency": 1, + "input_len": 262144, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 426.9566505359835, + "request_throughput": 0.04684316305857477, + "input_token_throughput": 12279.654136827025, + "output_token_throughput": 8.190527060791798, + "total_token_throughput": 12287.844663887816, + "total_input_tokens": 5242880, + "total_output_tokens": 3497, + "e2e_ms": { + "mean": 14470.753410700127, + "p50": 14486.497411038727, + "p90": 14829.92560529965, + "p95": 14834.793809620896, + "p99": 14835.151171508478 + }, + "ttft_ms": { + "mean": 13218.703388556605, + "p50": 13146.110309491633, + "p90": 13440.408597118221, + "p95": 13504.541694695945, + "p99": 13586.495378156542 + }, + "tpot_ms": { + "mean": 7.217646949924548, + "p50": 7.205249976773752, + "p90": 7.285532500031017, + "p95": 7.323570450965531, + "p99": 7.362167690467771 + }, + "itl_ms": { + "mean": 7.0786326117089216, + "p50": 7.069641003355816, + "p90": 7.148613351776059, + "p95": 7.149781891644509, + "p99": 7.1514094155602415 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_262144_256.jsonl" + }, + { + "name": "c1_i524288_o128", + "config": { + "concurrency": 1, + "input_len": 524288, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 1098.2779603750096, + "request_throughput": 0.018210326275846376, + "input_token_throughput": 9547.455542510945, + "output_token_throughput": 1.6225400711779119, + "total_token_throughput": 9549.078082582122, + "total_input_tokens": 10485760, + "total_output_tokens": 1782, + "e2e_ms": { + "mean": 36907.33525130199, + "p50": 36869.80330350343, + "p90": 36962.52591308439, + "p95": 37308.40163663088, + "p99": 37807.7560897154 + }, + "ttft_ms": { + "mean": 36239.33900470438, + "p50": 36162.923870026134, + "p90": 36277.070767228724, + "p95": 36595.43939542782, + "p99": 37102.25245506095 + }, + "tpot_ms": { + "mean": 9.374547555038637, + "p50": 7.512703494884909, + "p90": 7.905592262867882, + "p95": 9.796026421859272, + "p99": 37.00170689786954 + }, + "itl_ms": { + "mean": 9.320936322930454, + "p50": 7.497354151818467, + "p90": 7.850911301760415, + "p95": 9.684160583476693, + "p99": 36.89945372730697 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": true, + "overall": "⚠️" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_524288_128.jsonl" + }, + { + "name": "c1_i65536_o1024", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 114.83304778503953, + "request_throughput": 0.17416589027088075, + "input_token_throughput": 11414.135784792441, + "output_token_throughput": 39.37019949573259, + "total_token_throughput": 11453.505984288173, + "total_input_tokens": 1310720, + "total_output_tokens": 4521, + "e2e_ms": { + "mean": 4020.211619703332, + "p50": 3666.638041497208, + "p90": 4995.36795638851, + "p95": 5428.028983922561, + "p99": 7058.635749606762 + }, + "ttft_ms": { + "mean": 2462.7787359495414, + "p50": 2460.3703945176676, + "p90": 2469.169140793383, + "p95": 2475.2622663858347, + "p99": 2495.1480828761123 + }, + "tpot_ms": { + "mean": 6.921263477907232, + "p50": 6.920807158632908, + "p90": 6.927106867201686, + "p95": 6.928113336760625, + "p99": 6.932009929604653 + }, + "itl_ms": { + "mean": 6.869014879160817, + "p50": 6.86729956075302, + "p90": 6.892015080434886, + "p95": 6.89445152277582, + "p99": 6.902294996124454 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_1024.jsonl" + }, + { + "name": "c1_i65536_o128", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 96.3714261049754, + "request_throughput": 0.20753039368966497, + "input_token_throughput": 13600.711880845884, + "output_token_throughput": 24.89327072307531, + "total_token_throughput": 13625.605151568958, + "total_input_tokens": 1310720, + "total_output_tokens": 2399, + "e2e_ms": { + "mean": 3313.691728649428, + "p50": 3332.6328665134497, + "p90": 3340.3004989027977, + "p95": 3344.6828095882665, + "p99": 3373.650279486319 + }, + "ttft_ms": { + "mean": 2497.0991775451694, + "p50": 2515.7667085004505, + "p90": 2525.6168330204673, + "p95": 2528.4263670502696, + "p99": 2558.934195797192 + }, + "tpot_ms": { + "mean": 6.865009487508972, + "p50": 6.865337037847244, + "p90": 6.8708567147718815, + "p95": 6.871842963009778, + "p99": 6.876171570825193 + }, + "itl_ms": { + "mean": 6.864437613271855, + "p50": 6.864770382355132, + "p90": 6.870263566796199, + "p95": 6.871242446588812, + "p99": 6.8754964826359855 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_128.jsonl" + }, + { + "name": "c1_i65536_o256", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 105.9553989309934, + "request_throughput": 0.18875866828669668, + "input_token_throughput": 12370.488084836954, + "output_token_throughput": 32.66468754701286, + "total_token_throughput": 12403.152772383968, + "total_input_tokens": 1310720, + "total_output_tokens": 3461, + "e2e_ms": { + "mean": 3650.2073114039376, + "p50": 3586.5826304943766, + "p90": 4159.073864476522, + "p95": 4159.1440846154, + "p99": 4159.745420924155 + }, + "ttft_ms": { + "mean": 2460.456851005438, + "p50": 2457.9500810068566, + "p90": 2469.012875511544, + "p95": 2472.3303537844913, + "p99": 2483.1682235660264 + }, + "tpot_ms": { + "mean": 6.917922473178793, + "p50": 6.923507425054892, + "p90": 6.927156661085607, + "p95": 6.927327222851427, + "p99": 6.929477295310293 + }, + "itl_ms": { + "mean": 6.866275198167155, + "p50": 6.8652274387181045, + "p90": 6.89137497035926, + "p95": 6.892970948197751, + "p99": 6.895060622030097 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_256.jsonl" + }, + { + "name": "c1_i65536_o512", + "config": { + "concurrency": 1, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 20 + }, + "metrics": { + "success": 20, + "failed": 0, + "duration_s": 108.81008856697008, + "request_throughput": 0.18380648580844106, + "input_token_throughput": 12045.941853941993, + "output_token_throughput": 34.57399998056776, + "total_token_throughput": 12080.515853922561, + "total_input_tokens": 1310720, + "total_output_tokens": 3762, + "e2e_ms": { + "mean": 3756.9655780505855, + "p50": 3748.656399984611, + "p90": 4199.485603865469, + "p95": 4315.5037806223845, + "p99": 4593.5078689147485 + }, + "ttft_ms": { + "mean": 2461.8244336452335, + "p50": 2458.6545474885497, + "p90": 2466.431661415845, + "p95": 2468.7689636222785, + "p99": 2498.347155104275 + }, + "tpot_ms": { + "mean": 6.922369205894936, + "p50": 6.921919020527464, + "p90": 6.926635313933737, + "p95": 6.92835883957081, + "p99": 6.928915238310461 + }, + "itl_ms": { + "mean": 6.865564523081028, + "p50": 6.869272967554027, + "p90": 6.883120653382063, + "p95": 6.885109553744558, + "p99": 6.890476212814103 + } + }, + "slo_status": { + "ttft_p95_ok": true, + "tpot_mean_ok": true, + "overall": "✅" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_1_65536_512.jsonl" + }, + { + "name": "c32_i65536_o1024", + "config": { + "concurrency": 32, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 395.4516171090072, + "request_throughput": 0.40460069722232445, + "input_token_throughput": 26515.911293162255, + "output_token_throughput": 89.21951124623781, + "total_token_throughput": 26605.13080440849, + "total_input_tokens": 10485760, + "total_output_tokens": 35282, + "e2e_ms": { + "mean": 72731.84596702959, + "p50": 68012.98858498922, + "p90": 111259.59197211196, + "p95": 117936.92349772321, + "p99": 151338.19686311993 + }, + "ttft_ms": { + "mean": 18530.86177422847, + "p50": 15324.259108980186, + "p90": 36578.522250684895, + "p95": 54451.46387142016, + "p99": 68783.28264377428 + }, + "tpot_ms": { + "mean": 252.0000167805487, + "p50": 282.6309994048321, + "p90": 283.4716237898581, + "p95": 283.7050445944101, + "p99": 284.4103567225089 + }, + "itl_ms": { + "mean": 250.79581092386462, + "p50": 281.2108817644769, + "p90": 281.7336149556127, + "p95": 281.86471933110784, + "p99": 282.0100926023693 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_32_65536_1024.jsonl" + }, + { + "name": "c8_i131072_o512", + "config": { + "concurrency": 8, + "input_len": 131072, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 887.908358473971, + "request_throughput": 0.18019877667892276, + "input_token_throughput": 23619.014056859764, + "output_token_throughput": 33.12954509241995, + "total_token_throughput": 23652.143601952183, + "total_input_tokens": 20971520, + "total_output_tokens": 29416, + "e2e_ms": { + "mean": 41479.530511491976, + "p50": 40695.162369986065, + "p90": 58097.34920277842, + "p95": 66809.83657923935, + "p99": 79279.5650899608 + }, + "ttft_ms": { + "mean": 12820.608507814177, + "p50": 10825.261771999067, + "p90": 20686.60730908159, + "p95": 25478.948992854566, + "p99": 32945.62340407631 + }, + "tpot_ms": { + "mean": 156.62399342995585, + "p50": 157.0650420002514, + "p90": 211.635586992996, + "p95": 223.1889413108069, + "p99": 294.5085983403001 + }, + "itl_ms": { + "mean": 155.69947932134448, + "p50": 156.06387664204436, + "p90": 209.8603237579771, + "p95": 222.24342777714023, + "p99": 292.64473266343964 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_131072_512.jsonl" + }, + { + "name": "c8_i262144_o128", + "config": { + "concurrency": 8, + "input_len": 262144, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 2121.2106142269913, + "request_throughput": 0.07542862501577052, + "input_token_throughput": 19773.161476134148, + "output_token_throughput": 9.479963877763309, + "total_token_throughput": 19782.641440011914, + "total_input_tokens": 41943040, + "total_output_tokens": 20109, + "e2e_ms": { + "mean": 99110.04675216645, + "p50": 99465.57483251672, + "p90": 103144.93849380522, + "p95": 110875.081289644, + "p99": 129961.04960311025 + }, + "ttft_ms": { + "mean": 51391.65483183606, + "p50": 50290.368146525, + "p90": 53995.0529207359, + "p95": 61587.85037728085, + "p99": 80661.74531452528 + }, + "tpot_ms": { + "mean": 382.0441377238577, + "p50": 387.6563743937955, + "p90": 388.418829515108, + "p95": 388.6753990317422, + "p99": 389.2402474440457 + }, + "itl_ms": { + "mean": 381.6890187165073, + "p50": 387.61220860210665, + "p90": 388.2921533510144, + "p95": 388.5783935903698, + "p99": 388.8003312007251 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_262144_128.jsonl" + }, + { + "name": "c8_i65536_o1024", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 1024, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 422.40927311300766, + "request_throughput": 0.37877956329144086, + "input_token_throughput": 24823.697459867868, + "output_token_throughput": 76.88988397589192, + "total_token_throughput": 24900.58734384376, + "total_input_tokens": 10485760, + "total_output_tokens": 32479, + "e2e_ms": { + "mean": 19699.146988233042, + "p50": 17983.775263506686, + "p90": 27382.01765177655, + "p95": 35998.2133779646, + "p99": 56237.927606889854 + }, + "ttft_ms": { + "mean": 4665.850258549472, + "p50": 4614.468439016491, + "p90": 7052.845655445708, + "p95": 8696.151139616268, + "p99": 14818.984791898976 + }, + "tpot_ms": { + "mean": 73.86492770566875, + "p50": 74.79552028993388, + "p90": 93.63153691253392, + "p95": 99.00013507767628, + "p99": 113.60191243889423 + }, + "itl_ms": { + "mean": 73.42846517454711, + "p50": 74.52521800299834, + "p90": 93.57217559080777, + "p95": 98.94534214206857, + "p99": 112.92381913901038 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_1024.jsonl" + }, + { + "name": "c8_i65536_o128", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 128, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 402.3544815850328, + "request_throughput": 0.3976592962745114, + "input_token_throughput": 26060.99964064638, + "output_token_throughput": 46.478915622685236, + "total_token_throughput": 26107.478556269067, + "total_input_tokens": 10485760, + "total_output_tokens": 18701, + "e2e_ms": { + "mean": 18805.63353704165, + "p50": 18806.962396978633, + "p90": 20208.182343194494, + "p95": 21397.725383506626, + "p99": 22483.11250662489 + }, + "ttft_ms": { + "mean": 8659.910049839164, + "p50": 9297.093524510274, + "p90": 12907.319448608905, + "p95": 13193.446341765231, + "p99": 15765.712861008938 + }, + "tpot_ms": { + "mean": 87.86875209645738, + "p50": 81.1571685254522, + "p90": 131.84152213743047, + "p95": 143.77713837975247, + "p99": 166.11118562007195 + }, + "itl_ms": { + "mean": 87.83150954708688, + "p50": 81.15654301274425, + "p90": 131.8411181340271, + "p95": 143.77673574197482, + "p99": 166.0579600479071 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_128.jsonl" + }, + { + "name": "c8_i65536_o256", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 256, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 416.74979167600395, + "request_throughput": 0.38392340727164576, + "input_token_throughput": 25160.804418954576, + "output_token_throughput": 67.2513833475151, + "total_token_throughput": 25228.055802302093, + "total_input_tokens": 10485760, + "total_output_tokens": 28027, + "e2e_ms": { + "mean": 19445.902098786246, + "p50": 19457.487180974567, + "p90": 25653.07752349763, + "p95": 27248.316549730956, + "p99": 32918.893259076285 + }, + "ttft_ms": { + "mean": 5113.032919598118, + "p50": 4716.046326007927, + "p90": 9257.992207800271, + "p95": 10698.951332390427, + "p99": 14785.058989692472 + }, + "tpot_ms": { + "mean": 82.52785684036125, + "p50": 83.48555674335813, + "p90": 107.3290157292821, + "p95": 113.1300018626113, + "p99": 132.87966855654577 + }, + "itl_ms": { + "mean": 82.15261858594938, + "p50": 82.55044210092956, + "p90": 107.26047251854484, + "p95": 113.05118970898464, + "p99": 132.78087972009826 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_256.jsonl" + }, + { + "name": "c8_i65536_o512", + "config": { + "concurrency": 8, + "input_len": 65536, + "output_len": 512, + "dataset": "random", + "num_prompts": 160 + }, + "metrics": { + "success": 160, + "failed": 0, + "duration_s": 422.6590841940488, + "request_throughput": 0.37855568703816556, + "input_token_throughput": 24809.025505733218, + "output_token_throughput": 76.23401745035459, + "total_token_throughput": 24885.259523183573, + "total_input_tokens": 10485760, + "total_output_tokens": 32221, + "e2e_ms": { + "mean": 19689.273196897193, + "p50": 17639.091348508373, + "p90": 30207.329620694505, + "p95": 34072.42025379383, + "p99": 42942.271773040986 + }, + "ttft_ms": { + "mean": 4532.429489877904, + "p50": 4538.110625522677, + "p90": 7080.716864136048, + "p95": 9015.108168005827, + "p99": 14789.723641666926 + }, + "tpot_ms": { + "mean": 75.30331398641667, + "p50": 74.66872287350321, + "p90": 100.42571048979165, + "p95": 108.22462882297587, + "p99": 126.16376873402673 + }, + "itl_ms": { + "mean": 74.9958891435758, + "p50": 74.5155918840237, + "p90": 100.33134087725274, + "p95": 108.15634023765763, + "p99": 125.34686926273575 + } + }, + "slo_status": { + "ttft_p95_ok": false, + "tpot_mean_ok": false, + "overall": "❌" + }, + "raw_file": "/data/user1/qqt/sskj/experiments/dsv4_h200_vllm_tp8_custom_bench/results/20260711-193756/raw_outputs/vllm_0712_8_65536_512.jsonl" + } + ] +} \ No newline at end of file diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh index 9120605..0e8adfa 100755 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh +++ b/experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh @@ -71,7 +71,6 @@ data["config"] = { "gpu_groups": "0,1,2,3,4,5,6,7", "kv_cache_dtype": "fp8", "block_size": 256, - "max_num_seqs": 256, "client": "bench_client.py", "lb_strategy": "round_robin", "scenarios": [ diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md deleted file mode 100644 index fc5aa6f..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# dsv4_h200_vllm_tp8_custom_bench_no_fp8 - -> **Note:** This variant does NOT use `--kv-cache-dtype fp8`. - -## 目的 - -当 vLLM 使用 **TP=8**（Tensor Parallel = 8）部署时，单张 H200 上运行 **1 个服务**，独占全部 8 张 GPU： - -| 服务 | 端口 | GPU 组 | -|------|------|--------| -| 1 | 30005 | 0, 1, 2, 3, 4, 5, 6, 7 | - -TP=8 提供最大的单服务 tensor-parallel 宽度，适合极高并发、大 batch 场景。本实验使用 **自定义压测客户端 `bench_client.py`**，直接调用 OpenAI `/v1/chat/completions` API。 - -## 与多服务方案的区别 - -| 特性 | TP=2 (4 服务) | TP=4 (2 服务) | TP=8 (1 服务) | -|------|---------------|---------------|---------------| -| 服务数量 | 4 | 2 | 1 | -| 每服务 GPU | 2 | 4 | 8 | -| 单服务吞吐 | 较低 | 中等 | 最高 | -| 集群总吞吐 | 相近 | 相近 | 相近 | -| 适用场景 | 低并发、低延迟 | 中等并发 | 极高并发、大 batch | - -## 文件说明 - -| 文件 | 说明 | -|------|------| -| `bench_client.py` | 自定义异步压测客户端（复用 TP=2/TP=4 版本） | -| `config.env` | 实验配置（单服务、端口 30005） | -| `run_bench.sh` | 编排脚本：启动 1 个服务 → 运行压测 → 解析结果 | -| `start_server.sh` | 启动 1 个 vLLM TP=8 服务（占用全部 8 GPU） | -| `parse_results.py` | 解析 bench_client.py 输出的 JSONL，生成 `results.json` + `report.md` | - -## 快速运行 - -```bash -# 完整流程（自动启动服务、压测、生成报告） -bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh - -# 跳过服务管理（已有服务在运行） -SKIP_MANAGE_SERVER=1 bash experiments/dsv4_h200_vllm_tp8_custom_bench/run_bench.sh - -# 单独运行压测客户端（单服务） -python3 experiments/dsv4_h200_vllm_tp8_custom_bench/bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 32 --input-len 512 --output-len 256 --num-prompts 640 \ - --output-file /tmp/test.jsonl - -# 解析已有结果 -python3 experiments/dsv4_h200_vllm_tp8_custom_bench/parse_results.py \ - experiments/dsv4_h200_vllm_tp8_custom_bench/results/ -``` - -## 场景设计 - -本实验覆盖了从 **单并发** 到 **128 并发** 的梯度，以及不同输入/输出长度组合： - -| 并发 | 输入长度 | 输出长度 | 请求数 | 说明 | -|------|----------|----------|--------|------| -| 1 | 512 | 256 | 20 | 单并发基线，测纯延迟 | -| 2 | 512 | 256 | 40 | 低并发 | -| 4 | 512 | 256 | 80 | 中低并发 | -| 8 | 512 | 256 | 160 | 中等并发 | -| 16 | 512 | 256 | 320 | 中高并发 | -| 32 | 512 | 256 | 640 | 高并发 | -| 64 | 512 | 256 | 1280 | 很高并发 | -| 128 | 512 | 256 | 2560 | 极限并发，测吞吐上限 | -| 1 | 2048 | 512 | 20 | 长输入基线 | -| 8 | 2048 | 512 | 160 | 长输入中并发 | -| 32 | 2048 | 512 | 640 | 长输入高并发 | -| 128 | 2048 | 512 | 2560 | 长输入极限并发 | -| 1 | 4096 | 1024 | 20 | 超长输入基线 | -| 8 | 4096 | 1024 | 160 | 超长输入中并发 | -| 32 | 4096 | 1024 | 640 | 超长输入高并发 | -| 128 | 4096 | 1024 | 2560 | 超长输入极限并发 | - -## 输出格式 - -同 TP=2/TP=4 版本，输出 JSONL 原始数据、`results.json` 结构化结果、`report.md` 人类可读报告。 - -## 单独启动/停止服务 - -```bash -# 启动服务 -bash experiments/dsv4_h200_vllm_tp8_custom_bench/start_server.sh - -# 停止（run_bench.sh 会自动停止，但也可以手动） -port=30005 -pid_file="experiments/dsv4_h200_vllm_tp8_custom_bench/server_port${port}.pid" -[[ -f "$pid_file" ]] && kill "$(cat "$pid_file")" 2>/dev/null || true -rm -f "$pid_file" -pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true -``` - -## 注意事项 - -- 服务独占全部 8 张 GPU，确保无其他进程占用 GPU。 -- 模型文件路径：`/data/models/DeepSeek-V4-Flash` -- TP=8 相比 TP=2/TP=4 单服务延迟最低，但无法通过多服务并行提升低并发下的集群利用率。 -- 健康检查依赖 `/health` 端点（vLLM 默认启用）。 -- 如果服务频繁崩溃，检查 GPU 显存是否充足（OOM 是最常见原因）。 diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py deleted file mode 100755 index d8edc36..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/bench_client.py +++ /dev/null @@ -1,617 +0,0 @@ -#!/usr/bin/env python3 -"""Custom benchmark client for vLLM / OpenAI-compatible API with multi-service load balancing and health-check failover. - -Sends concurrent requests to multiple inference servers and records per-request -latency metrics (TTFT, TPOT, ITL, E2E) with fine-grained timing. - -Designed for TP=2 on 8-GPU setups where 4 independent services run on -(0,1), (2,3), (4,5), (6,7). Requests are distributed across services via -round-robin or random load balancing. Unhealthy services are automatically -skipped with periodic retry. - -Usage: - # Single service (backward compatible) - python3 bench_client.py \ - --host 127.0.0.1 --port 30005 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --output-file results.jsonl - - # Multi-service (4 services on ports 30005-30008) - python3 bench_client.py \ - --host 127.0.0.1 \ - --ports 30005,30006,30007,30008 \ - --model deepseek-v4-flash \ - --concurrency 64 --input-len 512 --output-len 256 --num-prompts 320 \ - --lb-strategy round_robin \ - --output-file results.jsonl - -The output JSONL contains one object per request with raw timing data. -A final summary line ("completed" field) aggregates all requests. -""" - -import argparse -import asyncio -import json -import random -import sys -import time -import urllib.request -from dataclasses import asdict, dataclass, field -from typing import Any - -# Try aiohttp first; fall back to urllib for stdlib-only environments. -try: - import aiohttp - HAS_AIOHTTP = True -except ImportError: - HAS_AIOHTTP = False - - -# --------------------------------------------------------------------------- -# Data structures -# --------------------------------------------------------------------------- - -@dataclass -class RequestResult: - request_id: int - input_tokens: int = 0 - output_tokens: int = 0 - first_token_time_ms: float = 0.0 # TTFT - e2e_latency_ms: float = 0.0 # total wall time - inter_token_latencies: list[float] = field(default_factory=list) - success: bool = False - error: str = "" - target_port: int = 0 # which service handled this request - retry_count: int = 0 # how many times this request was retried - - @property - def tpot_ms(self) -> float: - """Mean TPOT = (e2e - ttft) / (output_tokens - 1) if >1 token.""" - if self.output_tokens <= 1: - return 0.0 - return (self.e2e_latency_ms - self.first_token_time_ms) / (self.output_tokens - 1) - - @property - def mean_itl_ms(self) -> float: - if not self.inter_token_latencies: - return 0.0 - return sum(self.inter_token_latencies) / len(self.inter_token_latencies) - - def to_dict(self) -> dict[str, Any]: - d = asdict(self) - d["tpot_ms"] = self.tpot_ms - d["mean_itl_ms"] = self.mean_itl_ms - return d - - -# --------------------------------------------------------------------------- -# Health checker -# --------------------------------------------------------------------------- - -class HealthChecker: - """Periodically check service health and maintain a healthy-port list.""" - - def __init__( - self, - host: str, - ports: list[int], - check_interval: float = 5.0, - max_failures: int = 2, - recovery_interval: float = 10.0, - ): - self.host = host - self.all_ports = ports - self.check_interval = check_interval - self.max_failures = max_failures - self.recovery_interval = recovery_interval - - # port -> consecutive failure count - self._failures: dict[int, int] = {p: 0 for p in ports} - # port -> last healthy timestamp - self._last_healthy: dict[int, float] = {p: time.monotonic() for p in ports} - self._lock = asyncio.Lock() - self._task: asyncio.Task | None = None - - async def _check_one(self, port: int, session: aiohttp.ClientSession | None) -> bool: - url = f"http://{self.host}:{port}/health" - try: - if HAS_AIOHTTP and session is not None: - async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp: - return resp.status == 200 - else: - req = urllib.request.Request(url, method="GET") - with urllib.request.urlopen(req, timeout=5) as resp: - return resp.status == 200 - except Exception: - return False - - async def _check_all(self, session: aiohttp.ClientSession | None) -> None: - checks = [self._check_one(p, session) for p in self.all_ports] - results = await asyncio.gather(*checks, return_exceptions=True) - now = time.monotonic() - async with self._lock: - for port, ok in zip(self.all_ports, results): - if isinstance(ok, Exception): - ok = False - if ok: - self._failures[port] = 0 - self._last_healthy[port] = now - else: - self._failures[port] += 1 - - async def start(self, session: aiohttp.ClientSession | None) -> None: - async def _loop() -> None: - while True: - await asyncio.sleep(self.check_interval) - await self._check_all(session) - self._task = asyncio.create_task(_loop()) - - async def stop(self) -> None: - if self._task is not None: - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - - @property - async def healthy_ports(self) -> list[int]: - now = time.monotonic() - async with self._lock: - healthy = [] - for p in self.all_ports: - if self._failures[p] < self.max_failures: - healthy.append(p) - elif now - self._last_healthy[p] > self.recovery_interval: - # Give it another chance after recovery_interval. - self._failures[p] = max(0, self._failures[p] - 1) - healthy.append(p) - return healthy - - async def is_healthy(self, port: int) -> bool: - ports = await self.healthy_ports - return port in ports - - -# --------------------------------------------------------------------------- -# Load balancer with failover -# --------------------------------------------------------------------------- - -class LoadBalancer: - """Distribute requests across healthy backend ports with auto-failover.""" - - def __init__( - self, - ports: list[int], - health_checker: HealthChecker, - strategy: str = "round_robin", - ): - self.all_ports = ports - self.health_checker = health_checker - self.strategy = strategy - self._idx = 0 - self._lock = asyncio.Lock() - - def _next_round_robin(self, healthy: list[int]) -> int: - if not healthy: - return self.all_ports[self._idx % len(self.all_ports)] - for _ in range(len(self.all_ports)): - port = self.all_ports[self._idx % len(self.all_ports)] - self._idx += 1 - if port in healthy: - return port - # Fallback: all unhealthy, pick first healthy anyway. - return healthy[0] if healthy else self.all_ports[0] - - def _next_random(self, healthy: list[int]) -> int: - if healthy: - return random.choice(healthy) - return random.choice(self.all_ports) - - async def next_port(self) -> int: - healthy = await self.health_checker.healthy_ports - async with self._lock: - if self.strategy == "round_robin": - return self._next_round_robin(healthy) - elif self.strategy in ("random", "least_conn"): - return self._next_random(healthy) - else: - return self._next_round_robin(healthy) - - @property - def port_count(self) -> int: - return len(self.all_ports) - - -# --------------------------------------------------------------------------- -# Token helpers -# --------------------------------------------------------------------------- - -def make_prompt(token_len: int, vocab_size: int = 32000) -> str: - """Generate a random-ish prompt of approximately `token_len` tokens.""" - words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", - "dog", "hello", "world", "deep", "seek", "model", "test", - "benchmark", "performance", "latency", "throughput", "token"] - needed = max(token_len, 1) - tokens: list[str] = [] - while len(tokens) < needed: - tokens.extend(words) - return " ".join(tokens[:needed]) - - -# --------------------------------------------------------------------------- -# aiohttp-based async request (preferred) -# --------------------------------------------------------------------------- - -async def send_request_aiohttp( - session: aiohttp.ClientSession, - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": True, - } - headers = {"Content-Type": "application/json"} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - first_token_received = False - last_token_time = 0.0 - - try: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status != 200: - text = await resp.text() - result.error = f"HTTP {resp.status}: {text[:200]}" - return result - - async for line in resp.content: - line = line.decode("utf-8").strip() - if not line or not line.startswith("data: "): - continue - data_str = line[len("data: "):] - if data_str == "[DONE]": - break - try: - chunk = json.loads(data_str) - except json.JSONDecodeError: - continue - - choices = chunk.get("choices", []) - if not choices: - continue - - delta = choices[0].get("delta", {}) - content = delta.get("content", "") - if content: - now = time.perf_counter() - if not first_token_received: - result.first_token_time_ms = (now - start_time) * 1000 - first_token_received = True - else: - result.inter_token_latencies.append((now - last_token_time) * 1000) - last_token_time = now - result.output_tokens += 1 - - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.input_tokens = len(prompt.split()) # approximate - result.success = True - - except asyncio.TimeoutError: - result.error = "timeout" - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# urllib-based synchronous request (fallback, no aiohttp) -# --------------------------------------------------------------------------- - -def send_request_urllib( - host: str, - port: int, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, -) -> RequestResult: - """Blocking fallback using stdlib urllib. Used when aiohttp is absent.""" - url = f"http://{host}:{port}/v1/chat/completions" - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": max_tokens, - "temperature": temperature, - "stream": False, - } - data = json.dumps(payload).encode("utf-8") - headers = {"Content-Type": "application/json", "Content-Length": str(len(data))} - - result = RequestResult(request_id=request_id, target_port=port) - start_time = time.perf_counter() - - try: - req = urllib.request.Request(url, data=data, headers=headers, method="POST") - with urllib.request.urlopen(req, timeout=600) as resp: - body = json.loads(resp.read().decode("utf-8")) - - choices = body.get("choices", []) - if not choices: - result.error = "no choices in response" - return result - - content = choices[0].get("message", {}).get("content", "") - result.output_tokens = len(content.split()) if content else 0 - result.e2e_latency_ms = (time.perf_counter() - start_time) * 1000 - result.first_token_time_ms = result.e2e_latency_ms - result.input_tokens = len(prompt.split()) - result.success = True - - except Exception as e: - result.error = str(e)[:200] - - return result - - -# --------------------------------------------------------------------------- -# Unified send_request wrapper with retry & failover -# --------------------------------------------------------------------------- - -async def send_request_with_retry( - session: aiohttp.ClientSession | None, - host: str, - lb: LoadBalancer, - model: str, - request_id: int, - prompt: str, - max_tokens: int, - temperature: float, - max_retries: int = 2, -) -> RequestResult: - """Send a request, retrying on failure with a different port each time.""" - result = RequestResult(request_id=request_id) - for attempt in range(max_retries + 1): - port = await lb.next_port() - if HAS_AIOHTTP and session is not None: - result = await send_request_aiohttp( - session, host, port, model, request_id, prompt, max_tokens, temperature - ) - else: - loop = asyncio.get_event_loop() - result = await loop.run_in_executor( - None, send_request_urllib, - host, port, model, request_id, prompt, max_tokens, temperature - ) - result.retry_count = attempt - if result.success: - return result - # If failed, try another port on next iteration (unless last attempt). - if attempt < max_retries: - await asyncio.sleep(0.5 * (attempt + 1)) # exponential backoff-ish - return result - - -# --------------------------------------------------------------------------- -# Benchmark orchestration -# --------------------------------------------------------------------------- - -async def run_benchmark( - host: str, - ports: list[int], - lb: LoadBalancer, - health_checker: HealthChecker, - model: str, - concurrency: int, - input_len: int, - output_len: int, - num_prompts: int, - temperature: float, - output_file: str, -) -> dict[str, Any]: - """Run the benchmark and write a JSONL file.""" - prompts = [make_prompt(input_len) for _ in range(num_prompts)] - results: list[RequestResult] = [] - semaphore = asyncio.Semaphore(concurrency) - - start = time.perf_counter() - - if HAS_AIOHTTP: - connector = aiohttp.TCPConnector(limit=concurrency * 2) - timeout = aiohttp.ClientTimeout(total=600, sock_read=300) - async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session: - await health_checker.start(session) - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - session, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - else: - await health_checker.start(None) - from concurrent.futures import ThreadPoolExecutor - - async def bounded_send(idx: int, prompt: str) -> RequestResult: - async with semaphore: - return await send_request_with_retry( - None, host, lb, model, idx, prompt, output_len, temperature - ) - - tasks = [bounded_send(i, p) for i, p in enumerate(prompts)] - results = await asyncio.gather(*tasks) - await health_checker.stop() - - duration = time.perf_counter() - start - - # Write JSONL - with open(output_file, "w", encoding="utf-8") as f: - for r in results: - f.write(json.dumps(r.to_dict(), ensure_ascii=False) + "\n") - - # Compute summary - successful = [r for r in results if r.success] - failed = len(results) - len(successful) - retried = [r for r in results if r.retry_count > 0] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - ttfts = [r.first_token_time_ms for r in successful] - tpots = [r.tpot_ms for r in successful if r.output_tokens > 1] - e2es = [r.e2e_latency_ms for r in successful] - itls = [r.mean_itl_ms for r in successful if r.inter_token_latencies] - - total_input_tokens = sum(r.input_tokens for r in successful) - total_output_tokens = sum(r.output_tokens for r in successful) - - # Per-port breakdown - per_port_stats: dict[int, dict[str, Any]] = {} - for port in ports: - port_results = [r for r in successful if r.target_port == port] - port_failed = [r for r in results if r.target_port == port and not r.success] - if port_results or port_failed: - port_ttfts = [r.first_token_time_ms for r in port_results] - port_e2es = [r.e2e_latency_ms for r in port_results] - per_port_stats[port] = { - "count": len(port_results), - "failed": len(port_failed), - "mean_ttft_ms": sum(port_ttfts) / len(port_ttfts) if port_ttfts else 0.0, - "p95_ttft_ms": percentile(port_ttfts, 95) if port_ttfts else 0.0, - "mean_e2e_ms": sum(port_e2es) / len(port_e2es) if port_e2es else 0.0, - } - - # Health-check stats - healthy_at_end = await health_checker.healthy_ports - - summary = { - "completed": len(successful), - "failed": failed, - "retried": len(retried), - "duration": duration, - "request_throughput": len(successful) / duration if duration > 0 else 0.0, - "input_throughput": total_input_tokens / duration if duration > 0 else 0.0, - "output_throughput": total_output_tokens / duration if duration > 0 else 0.0, - "total_throughput": (total_input_tokens + total_output_tokens) / duration if duration > 0 else 0.0, - "total_input_tokens": total_input_tokens, - "total_output_tokens": total_output_tokens, - "mean_ttft_ms": sum(ttfts) / len(ttfts) if ttfts else 0.0, - "median_ttft_ms": percentile(ttfts, 50), - "p90_ttft_ms": percentile(ttfts, 90), - "p95_ttft_ms": percentile(ttfts, 95), - "p99_ttft_ms": percentile(ttfts, 99), - "mean_tpot_ms": sum(tpots) / len(tpots) if tpots else 0.0, - "median_tpot_ms": percentile(tpots, 50), - "p90_tpot_ms": percentile(tpots, 90), - "p95_tpot_ms": percentile(tpots, 95), - "p99_tpot_ms": percentile(tpots, 99), - "mean_e2e_latency_ms": sum(e2es) / len(e2es) if e2es else 0.0, - "median_e2e_latency_ms": percentile(e2es, 50), - "p90_e2e_latency_ms": percentile(e2es, 90), - "p95_e2e_latency_ms": percentile(e2es, 95), - "p99_e2e_latency_ms": percentile(e2es, 99), - "mean_itl_ms": sum(itls) / len(itls) if itls else 0.0, - "median_itl_ms": percentile(itls, 50), - "p90_itl_ms": percentile(itls, 90), - "p95_itl_ms": percentile(itls, 95), - "p99_itl_ms": percentile(itls, 99), - "per_port_stats": per_port_stats, - "healthy_ports_at_end": healthy_at_end, - "lb_strategy": lb.strategy, - "num_services": lb.port_count, - } - - # Append summary as the last line - with open(output_file, "a", encoding="utf-8") as f: - f.write(json.dumps(summary, ensure_ascii=False) + "\n") - - return summary - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser(description="Custom vLLM benchmark client with multi-service LB and health-check failover") - parser.add_argument("--host", default="127.0.0.1") - parser.add_argument("--port", type=int, default=30005, - help="Single service port (backward compatible)") - parser.add_argument("--ports", default="", - help="Comma-separated list of ports for multi-service, e.g. 30005,30006,30007,30008") - parser.add_argument("--model", default="deepseek-v4-flash") - parser.add_argument("--concurrency", type=int, default=1) - parser.add_argument("--input-len", type=int, default=512) - parser.add_argument("--output-len", type=int, default=256) - parser.add_argument("--num-prompts", type=int, default=10) - parser.add_argument("--temperature", type=float, default=0.0) - parser.add_argument("--lb-strategy", default="round_robin", - choices=["round_robin", "random", "least_conn"], - help="Load balancing strategy across services") - parser.add_argument("--health-check-interval", type=float, default=5.0, - help="Seconds between health checks (default: 5)") - parser.add_argument("--health-max-failures", type=int, default=2, - help="Consecutive failures before marking a port unhealthy (default: 2)") - parser.add_argument("--health-recovery-interval", type=float, default=10.0, - help="Seconds before retrying an unhealthy port (default: 10)") - parser.add_argument("--request-max-retries", type=int, default=2, - help="Max retries per request on failure (default: 2)") - parser.add_argument("--output-file", required=True) - args = parser.parse_args() - - # Determine ports: --ports takes precedence, else fall back to --port. - if args.ports: - ports = [int(p.strip()) for p in args.ports.split(",")] - else: - ports = [args.port] - - health_checker = HealthChecker( - host=args.host, - ports=ports, - check_interval=args.health_check_interval, - max_failures=args.health_max_failures, - recovery_interval=args.health_recovery_interval, - ) - lb = LoadBalancer(ports, health_checker=health_checker, strategy=args.lb_strategy) - - summary = asyncio.run(run_benchmark( - host=args.host, - ports=ports, - lb=lb, - health_checker=health_checker, - model=args.model, - concurrency=args.concurrency, - input_len=args.input_len, - output_len=args.output_len, - num_prompts=args.num_prompts, - temperature=args.temperature, - output_file=args.output_file, - )) - - print(json.dumps(summary, indent=2, ensure_ascii=False)) - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env deleted file mode 100644 index 618099b..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/config.env +++ /dev/null @@ -1,83 +0,0 @@ -# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench -# Custom benchmark client for multi-service vLLM TP=2 on 8x H200. -# -# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs: -# Service 1: port 30005, GPUs 0,1 -# Service 2: port 30006, GPUs 2,3 -# Service 3: port 30007, GPUs 4,5 -# Service 4: port 30008, GPUs 6,7 -# -# bench_client.py distributes requests across all 4 services via round-robin -# load balancing, giving accurate cluster-wide throughput even at low -# per-service concurrency. - -EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench_no_fp8" -MODEL_NAME="DeepSeek-V4-Flash" -MODEL_PATH="/data/models/DeepSeek-V4-Flash" -SERVED_MODEL_NAME="deepseek-v4-flash" - -# Primary port (backward compatible); multi-service ports are in PORTS. -PORT="30005" -PORTS="30005" - -BACKEND="vllm" -ENGINE="vllm" - -# Native virtual environments on the host. -VENV_SERVER="/data/user1/yy/envs/vllm" -VENV_CLIENT="/data/user1/yy/envs/vllm" - -# Load balancing strategy: round_robin | random | least_conn -LB_STRATEGY="round_robin" - -# Benchmark scenarios: "concurrency input_len output_len num_prompts". -# For single-service TP=8 we cover standard concurrency gradients. -SCENARIOS=( - "1 512 256 20" - "2 512 256 40" - "4 512 256 80" - "8 512 256 160" - "16 512 256 320" - "32 512 256 640" - "64 512 256 1280" - "128 512 256 2560" - "1 2048 512 20" - "8 2048 512 160" - "32 2048 512 640" - "128 2048 512 2560" - "1 4096 1024 20" - "8 4096 1024 160" - "32 4096 1024 640" - "128 4096 1024 2560" - "1 8192 1024 20" - "8 8192 1024 160" - "32 8192 1024 640" - "128 8192 1024 2560" - "1 16384 1024 20" - "8 16384 1024 160" - "32 16384 1024 640" - "128 16384 1024 2560" - "1 32768 1024 20" - "8 32768 1024 160" - "32 32768 1024 640" - "128 32768 1024 2560" - "1 65536 1024 20" - "8 65536 1024 160" - "32 65536 1024 640" - "128 65536 1024 2560" - "1 131072 1024 20" - "8 131072 1024 160" - "32 131072 1024 640" - "128 131072 1024 2560" - "1 262144 1024 20" - "8 262144 1024 160" - "32 262144 1024 640" - "128 262144 1024 2560" - "1 524288 1024 20" - "8 524288 1024 160" - "32 524288 1024 640" - "128 524288 1024 2560" -) - -# Server start script bundled with this experiment. -SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py deleted file mode 100755 index eea1a2e..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/parse_results.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Parse custom bench_client.py JSONL outputs for vLLM TP=2 benchmark. - -Reads JSONL files produced by bench_client.py (one request per line, -last line is the summary) and generates: - - results.json (appended scenarios, following the standard schema) - - report.md (human-readable summary) - -Usage: - python3 parse_results.py -""" - -import json -import sys -from pathlib import Path - - -def parse_jsonl(path: Path) -> tuple[dict | None, list[dict]]: - """Read the summary JSON (last line) and all per-request lines.""" - requests: list[dict] = [] - summary: dict | None = None - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - # The summary line has "completed" and "duration" keys. - if "completed" in obj and "duration" in obj: - summary = obj - else: - requests.append(obj) - return summary, requests - - -def compute_metrics(summary: dict, requests: list[dict]) -> dict: - completed = summary.get("completed", 0) - total = len(requests) - failed = total - completed if total > 0 else 0 - duration_s = summary.get("duration", 0.0) - - # Extract per-request metrics for percentiles. - ttfts = [r["first_token_time_ms"] for r in requests if r.get("success")] - tpots = [r["tpot_ms"] for r in requests if r.get("success") and r.get("output_tokens", 0) > 1] - e2es = [r["e2e_latency_ms"] for r in requests if r.get("success")] - itls = [r["mean_itl_ms"] for r in requests if r.get("success") and r.get("inter_token_latencies")] - - def percentile(values: list[float], p: float) -> float: - if not values: - return 0.0 - s = sorted(values) - k = (len(s) - 1) * p / 100.0 - f = int(k) - c = min(f + 1, len(s) - 1) - return s[f] + (k - f) * (s[c] - s[f]) - - return { - "success": completed, - "failed": failed, - "duration_s": duration_s, - "request_throughput": summary.get("request_throughput", 0.0), - "input_token_throughput": summary.get("input_throughput", 0.0), - "output_token_throughput": summary.get("output_throughput", 0.0), - "total_token_throughput": summary.get("total_throughput", 0.0), - "total_input_tokens": summary.get("total_input_tokens", 0), - "total_output_tokens": summary.get("total_output_tokens", 0), - "e2e_ms": { - "mean": summary.get("mean_e2e_latency_ms", 0.0), - "p50": percentile(e2es, 50), - "p90": percentile(e2es, 90), - "p95": percentile(e2es, 95), - "p99": percentile(e2es, 99), - }, - "ttft_ms": { - "mean": summary.get("mean_ttft_ms", 0.0), - "p50": percentile(ttfts, 50), - "p90": percentile(ttfts, 90), - "p95": percentile(ttfts, 95), - "p99": percentile(ttfts, 99), - }, - "tpot_ms": { - "mean": summary.get("mean_tpot_ms", 0.0), - "p50": percentile(tpots, 50), - "p90": percentile(tpots, 90), - "p95": percentile(tpots, 95), - "p99": percentile(tpots, 99), - }, - "itl_ms": { - "mean": summary.get("mean_itl_ms", 0.0), - "p50": percentile(itls, 50), - "p90": percentile(itls, 90), - "p95": percentile(itls, 95), - "p99": percentile(itls, 99), - }, - } - - -def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: - return f"c{concurrency}_i{input_len}_o{output_len}" - - -def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: - ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms - tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms - if ttft_ok and tpot_ok: - mark = "✅" - elif ttft_ok or tpot_ok: - mark = "⚠️" - else: - mark = "❌" - return { - "ttft_p95_ok": ttft_ok, - "tpot_mean_ok": tpot_ok, - "overall": mark, - } - - -def append_scenario(results_json: Path, scenario: dict) -> None: - with open(results_json, "r", encoding="utf-8") as f: - data = json.load(f) - data["scenarios"].append(scenario) - with open(results_json, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) - - -def generate_report(result_root: Path, scenarios: list[dict]) -> None: - report_path = result_root / "report.md" - with open(report_path, "w", encoding="utf-8") as f: - f.write("# H200 vLLM TP=2 Custom Benchmark Report\n\n") - f.write("- **Client**: `bench_client.py` (async OpenAI API, per-request timing)\n") - f.write("- **Backend**: vLLM (TP=2, FP8 KV cache, no speculative decoding)\n") - f.write(f"- **Result root**: `{result_root}`\n\n") - - f.write("## Results\n\n") - f.write("| Scenario | Concurrency | Input | Output | Duration(s) | Success | Failed | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") - f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") - - for s in scenarios: - cfg = s["config"] - m = s["metrics"] - slo = s.get("slo_status", {}).get("overall", "") - f.write( - f"| {s['name']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " - f"{m['duration_s']:.2f} | {m['success']} | {m['failed']} | {m['request_throughput']:.2f} | " - f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " - f"{m['total_token_throughput']:.2f} | " - f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " - f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " - f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" - ) - f.write("\n") - f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") - - -def main() -> None: - result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results") - raw_dir = result_root / "raw_outputs" - results_json = result_root / "results.json" - - if not raw_dir.exists(): - raise SystemExit(f"raw_outputs directory not found: {raw_dir}") - - scenarios = [] - for jsonl_path in sorted(raw_dir.glob("vllm_*.jsonl")): - parts = jsonl_path.stem.split("_") - if len(parts) < 5: - continue - concurrency, input_len, output_len = int(parts[2]), int(parts[3]), int(parts[4]) - - summary, requests = parse_jsonl(jsonl_path) - if summary is None: - continue - - metrics = compute_metrics(summary, requests) - scenario = { - "name": scenario_name(concurrency, input_len, output_len), - "config": { - "concurrency": concurrency, - "input_len": input_len, - "output_len": output_len, - "dataset": "random", - "num_prompts": metrics["success"] + metrics["failed"], - }, - "metrics": metrics, - "slo_status": slo_status(metrics), - "raw_file": str(jsonl_path), - } - scenarios.append(scenario) - - if not scenarios: - print("No benchmark outputs found to parse") - return - - if results_json.exists(): - for s in scenarios: - append_scenario(results_json, s) - - generate_report(result_root, scenarios) - print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") - - -if __name__ == "__main__": - main() diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh deleted file mode 100755 index 2319cae..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/run_bench.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env bash -# Custom benchmark orchestrator for single-service vLLM TP=8 on 8x H200. -# -# This runs 1 vLLM service (TP=8) on all 8 GPUs (0,1,2,3,4,5,6,7). -# bench_client.py sends requests to the single service. -# -# Usage: -# bash run_bench.sh -# SKIP_MANAGE_SERVER=1 bash run_bench.sh # skip server start/stop - -set -Eeuo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" - -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" -RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}" -RAW_DIR="${RESULT_ROOT}/raw_outputs" -LOG_DIR="${RESULT_ROOT}/logs" - -ensure_result_root "$RESULT_ROOT" -log_init "${LOG_DIR}/orchestrator.log" - -log "experiment=${EXPERIMENT_NAME}" -log "run_id=${RUN_ID}" -log "result_root=${RESULT_ROOT}" -log "platform=${PLATFORM}" -log "chip=${CHIP}" -log "accelerator=${ACCELERATOR}" -log "engine=${ENGINE}" -log "hardware=${HARDWARE}" -log "model=${MODEL_PATH}" -log "services=${PORTS}" -log "lb_strategy=${LB_STRATEGY}" -log "server_start_script=${SERVER_START_SCRIPT}" - -# Write initial metadata for this run. -METADATA_JSON="${RESULT_ROOT}/results.json" -write_metadata_json \ - "$METADATA_JSON" \ - "$EXPERIMENT_NAME" \ - "$RUN_ID" \ - "$MODEL_PATH" \ - "$BACKEND" \ - "$ENGINE" \ - "$HARDWARE" \ - "$ACCELERATOR" \ - "$CHIP" \ - "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ - "$VENV_SERVER" \ - "H200 1x vLLM TP=8 single-service (no FP8 KV cache) benchmark using bench_client.py" - -# Update metadata with config. -"${VENV_CLIENT}/bin/python" - "$METADATA_JSON" <<'PY' -import json, sys -path = sys.argv[1] -with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - -data["config"] = { - "tp": 8, - "num_services": 1, - "ports": "30005", - "gpu_groups": "0,1,2,3,4,5,6,7", - "kv_cache_dtype": "fp8", - "block_size": 256, - "max_num_seqs": 256, - "client": "bench_client.py", - "lb_strategy": "round_robin", - "scenarios": [ - "1 512 256 20", "2 512 256 40", "4 512 256 80", - "8 512 256 160", "16 512 256 320", "32 512 256 640", - "64 512 256 1280", "128 512 256 2560", - "1 2048 512 20", "8 2048 512 160", "32 2048 512 640", "128 2048 512 2560", - "1 4096 1024 20", "8 4096 1024 160", "32 4096 1024 640", "128 4096 1024 2560" - ] -} -with open(path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2, ensure_ascii=False) -PY - -# --------------------------------------------------------------------------- -# Server helpers (single-service) -# --------------------------------------------------------------------------- - -is_server_healthy() { - local port="$1" - curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 -} - -are_all_services_healthy() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - for p in "${PORT_LIST[@]}"; do - if ! is_server_healthy "$p"; then - return 1 - fi - done - return 0 -} - -stop_all_servers() { - log "stopping all vllm services" - # Kill by PID files first. - for pid_file in "${SCRIPT_DIR}"/server_port*.pid; do - [[ -f "$pid_file" ]] || continue - local pid - pid="$(cat "$pid_file")" - if kill -0 "$pid" 2>/dev/null; then - log "killing pid=${pid}" - kill "$pid" 2>/dev/null || true - sleep 2 - kill -9 "$pid" 2>/dev/null || true - fi - rm -f "$pid_file" - done - # Fallback: kill any remaining vllm processes for this model. - pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true - sleep 2 -} - -start_all_servers() { - log "starting all vllm services with ${SERVER_START_SCRIPT}" - if [[ ! -x "${SERVER_START_SCRIPT}" ]]; then - log "error: start script not found or not executable: ${SERVER_START_SCRIPT}" - exit 1 - fi - bash "${SERVER_START_SCRIPT}" >> "${LOG_DIR}/server.outer.log" 2>&1 - - if ! are_all_services_healthy "$PORTS"; then - log "error: not all vllm services became healthy" - exit 1 - fi - log "all vllm services are healthy" -} - -# --------------------------------------------------------------------------- -# Pre-benchmark health check with detailed reporting -# --------------------------------------------------------------------------- - -check_services_detailed() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy_count=0 - local unhealthy_ports="" - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - ((healthy_count++)) - else - unhealthy_ports="${unhealthy_ports}${p} " - fi - done - echo "$healthy_count" - if [[ -n "$unhealthy_ports" ]]; then - echo "unhealthy: ${unhealthy_ports}" >&2 - fi -} - -wait_for_all_services() { - local ports="$1" - local max_wait="${2:-240}" - local interval="${3:-5}" - local elapsed=0 - while (( elapsed < max_wait )); do - local healthy_count - healthy_count="$(check_services_detailed "$ports" | head -1)" - IFS=',' read -ra PORT_LIST <<< "$ports" - if (( healthy_count == ${#PORT_LIST[@]} )); then - return 0 - fi - log "waiting for services... ${healthy_count}/${#PORT_LIST[@]} healthy after ${elapsed}s" - sleep "$interval" - ((elapsed += interval)) - done - return 1 -} - -# --------------------------------------------------------------------------- -# Mid-benchmark health check (called between scenarios) -# --------------------------------------------------------------------------- - -check_and_report_services() { - local ports="$1" - IFS=',' read -ra PORT_LIST <<< "$ports" - local healthy=() - local unhealthy=() - for p in "${PORT_LIST[@]}"; do - if is_server_healthy "$p"; then - healthy+=("$p") - else - unhealthy+=("$p") - fi - done - log "service health check: healthy=[${healthy[*]}] unhealthy=[${unhealthy[*]}]" - if [[ ${#unhealthy[@]} -gt 0 ]]; then - log "WARNING: ${#unhealthy[@]} service(s) unhealthy: ${unhealthy[*]}" - fi - return ${#unhealthy[@]} -} - -on_exit() { - local code=$? - log "orchestrator exiting with code=${code}" - if [[ -z "${SKIP_MANAGE_SERVER:-}" ]]; then - stop_all_servers - fi - exit "$code" -} -trap on_exit EXIT - -if [[ -n "${SKIP_MANAGE_SERVER:-}" ]]; then - log "SKIP_MANAGE_SERVER is set, assuming services are already running" - if ! wait_for_all_services "$PORTS"; then - log "error: not all healthy services found on ports ${PORTS}" - exit 1 - fi -else - stop_all_servers - start_all_servers -fi - -log "===== BENCHMARK START =====" - -for scenario in "${SCENARIOS[@]}"; do - read -r concurrency input_len output_len num_prompts <<< "$scenario" - # Fallback for legacy 3-field scenarios. - if [[ -z "${num_prompts:-}" ]]; then - num_prompts="$NUM_PROMPTS" - fi - - output_file="${RAW_DIR}/vllm_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" - detail_log="${LOG_DIR}/vllm_c${concurrency}_i${input_len}_o${output_len}.log" - - log "running scenario: concurrency=${concurrency} input=${input_len} output=${output_len} num_prompts=${num_prompts}" - - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/bench_client.py" \ - --host 127.0.0.1 \ - --port "$PORT" \ - --model "$SERVED_MODEL_NAME" \ - --concurrency "$concurrency" \ - --input-len "$input_len" \ - --output-len "$output_len" \ - --num-prompts "$num_prompts" \ - --output-file "$output_file" \ - > "$detail_log" 2>&1 || { - log "ERROR: scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" - # After failure, check service health before continuing. - check_and_report_services "$PORTS" - continue - } - - log "finished scenario: output=${output_file}" - - # Between scenarios, report service health so we can spot degradation early. - check_and_report_services "$PORTS" || true -done - -log "===== BENCHMARK DONE =====" - -log "parsing results" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_results.py" "$RESULT_ROOT" >> "${LOG_DIR}/parse.log" 2>&1 || { - log "WARNING: parser failed; see ${LOG_DIR}/parse.log" -} - -log "all results saved to ${RESULT_ROOT}" diff --git a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh b/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh deleted file mode 100755 index b9dad2f..0000000 --- a/experiments/dsv4_h200_vllm_tp8_custom_bench_no_fp8/start_server.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# Start 1 vLLM TP=8 service on 8x H200. -# Uses all 8 GPUs: (0,1,2,3,4,5,6,7). -# This maximizes single-service throughput for highest-concurrency workloads. -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "${SCRIPT_DIR}/config.env" - -VENV="${VENV_SERVER}" -export PATH="$VENV/bin:$PATH" -VLLM="$VENV/bin/vllm" - -TP=8 -LOG_DIR="${SCRIPT_DIR}/logs" -mkdir -p "$LOG_DIR" - -# 1 service on all 8 GPUs. -# Port and GPU mapping. -SERVICES=( - "30005:0,1,2,3,4,5,6,7" -) - -# Kill any existing vLLM processes for this model. -pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true -sleep 2 - -# Remove old PID files. -rm -f "${SCRIPT_DIR}"/*.pid - -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - GPUS="${svc##*:}" - LOG="${LOG_DIR}/server_port${PORT}_$(date +%Y%m%d_%H%M%S).log" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - - echo "=== Starting service on port ${PORT}, GPUs ${GPUS} ===" - echo "Log: ${LOG}" - - CUDA_VISIBLE_DEVICES="${GPUS}" \ - nohup "$VLLM" serve "$MODEL_PATH" \ - --trust-remote-code \ - --tensor-parallel-size "$TP" \ - --block-size 256 \ - --served-model-name "$SERVED_MODEL_NAME" \ - --port "$PORT" \ - > "$LOG" 2>&1 & - PID=$! - echo $PID > "$PID_FILE" - echo "PID: $PID" -done - -echo "" -echo "Waiting for service to become healthy..." - -MAX_WAIT=240 -all_healthy=0 -for i in $(seq 1 $MAX_WAIT); do - all_healthy=1 - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - if ! curl -s "http://127.0.0.1:${PORT}/health" > /dev/null 2>&1; then - all_healthy=0 - break - fi - done - - if [[ "$all_healthy" -eq 1 ]]; then - echo "Service is healthy!" - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - echo " - http://127.0.0.1:${PORT}" - done - exit 0 - fi - - # Check if any process died early. - for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - PID_FILE="${SCRIPT_DIR}/server_port${PORT}.pid" - if [[ -f "$PID_FILE" ]]; then - PID="$(cat "$PID_FILE")" - if ! kill -0 "$PID" 2>/dev/null; then - echo "ERROR: Service on port ${PORT} (PID ${PID}) exited early" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - tail -200 $LOG 2>/dev/null || true - exit 1 - fi - fi - done - - if (( i % 10 == 0 )); then - echo "Waiting... (${i}/${MAX_WAIT})" - fi - sleep 5 -done - -echo "ERROR: Service did not become healthy after ${MAX_WAIT} retries" -for svc in "${SERVICES[@]}"; do - PORT="${svc%%:*}" - LOG="${LOG_DIR}/server_port${PORT}_*.log" - echo "--- Last 50 lines of port ${PORT} ---" - tail -50 $LOG 2>/dev/null || true -done -exit 1