diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/TP8_vs_TP16_report.md b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/TP8_vs_TP16_report.md new file mode 100644 index 0000000..49327c5 --- /dev/null +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/TP8_vs_TP16_report.md @@ -0,0 +1,110 @@ +# GLM-5.2-w4a8c8 测速报告:TP=8 vs TP=16 + +## 实验环境 + +| 项 | 值 | +|---|---| +| 机器 | 910c.2 (NPU-NODE62) | +| NPU | 8x Ascend910 (16 dies, 64GB HBM/die) | +| 模型 | GLM-5.2-w4a8c8 (391GB, 99 shards) | +| 引擎 | vLLM-Ascend 0.23.0 (sglang 0.5.2 bench_serving) | +| 镜像 | local/vllm-ascend:0.23-a3-20260718-sglang | +| 数据集 | random (ShareGPT tokenizer 种子) | +| ISL/OSL | 4096 / 512 | +| 投机解码 | 无 (最基本状态) | +| 日期 | 2026-07-28 | + +## 配置差异 + +| 参数 | TP=8 | TP=16 | +|---|---|---| +| tensor-parallel-size | 8 | 16 | +| 使用 die 数 | 8/16 (50%) | 16/16 (100%) | +| 每 die 权重 | 56.76 GB | ~24 GB | +| KV cache 可用 | 2.83 GiB | 23.10 GiB | +| max-model-len | 16384 | 131072 | +| max-num-seqs | 64 | 256 | +| gpu-memory-utilization | 0.95 | 0.92 | +| kv-cache-dtype | bf16 (fp8 未生效) | bf16 | + +## 测速结果对比 (ISL=4096, OSL=512) + +### TTFT (Time To First Token, ms) + +| 并发 | TP=8 | TP=16 | 倍数 | +|---|---|---|---| +| 1 | 431* | 902 | 0.48x | +| 4 | 684 | 539 | 1.27x | +| 8 | 935 | 558 | 1.68x | +| 16 | 5320 | 598 | 8.90x | +| 32 | - | 816 | - | +| 64 | - | 2148 | - | + +### TPOT (Time Per Output Token, ms) + +| 并发 | TP=8 | TP=16 | 倍数 | +|---|---|---|---| +| 1 | 43* | 50.1 | 0.86x | +| 4 | 55.1 | 52.0 | 1.06x | +| 8 | 69.6 | 59.3 | 1.17x | +| 16 | 88.3 | 72.1 | 1.23x | +| 32 | - | 105.1 | - | +| 64 | - | 175.5 | - | + +### 输出后吐 (tok/s) + +| 并发 | TP=8 | TP=16 | 倍数 | +|---|---|---|---| +| 1 | 20* | 19.5 | 1.03x | +| 4 | 62.6 | 66.6 | 1.06x | +| 8 | 104.0 | 119.4 | 1.15x | +| 16 | 145.0 | 192.8 | 1.33x | +| 32 | - | 275.1 | - | +| 64 | - | 329.5 | - | + +### 总吞吐 (input+output tok/s) + +| 并发 | TP=8 | TP=16 | 倍数 | +|---|---|---|---| +| 1 | 120* | 157.0 | 1.31x | +| 4 | 497 | 529.2 | 1.06x | +| 8 | 990 | 1137.3 | 1.15x | +| 16 | 1343 | 1785.1 | 1.33x | +| 32 | - | 2431.8 | - | +| 64 | - | 2925.4 | - | + +### SLO 达标情况 (TTFT P95 < 3000ms 且 TPOT mean < 50ms) + +| 并发 | TP=8 | TP=16 | +|---|---|---| +| 1 | PASS/PASS* | PASS/FAIL | +| 4 | PASS/FAIL | PASS/FAIL | +| 8 | PASS/FAIL | PASS/FAIL | +| 16 | FAIL/FAIL | PASS/FAIL | +| 32 | - | PASS/FAIL | +| 64 | - | PASS/FAIL | + +*TP=8 并发1数据来自 smoke test (ISL=1024/OSL=128),非同场景 + +## 关键结论 + +### 1. TP=16 吞吐优势显著 +- TP=16 最大输出吒吐 **329.5 tok/s**(并发64),是 TP=8 的 **2.27倍**(145 tok/s) +- TP=16 在并发32时仍能保持 275 tok/s,TP=8 在并发16已严重过载 + +### 2. TP=16 延迟更稳定 +- TP=8 在并发16时 TTFT 暴涨到 5320ms(过载-,TP=16 在并发64时 TTFT 仅 2148ms +- TP=16 在并发1-32范围内 TTFT 均保持在 1000ms 以内 + +### 3. TP=8 单请求延迟更低 +- 并发1时 TP=8 的 TTFT (431ms) 优于 TP=16 (902ms),因为 TP=8 通信开销更小 +- 但随着并发增加,TP=8 快速过载,优势消失 + +### 4. 显存边界决定可行性 +- TP=8 KV cache 仅 2.83 GiB,max-len 限制在 16384,无法支持长上下文 +- TP=16 KV cache 23.10 GiB,max-len 可达 131072,支持 128K 上下文 +- **TP=2/4 在 64GB die 上必然 OOM(权重 195GB/98GB > 64GB),不可行** + +### 5. 两者均未完全达到 S2 SLO +- TPOT 在并发≥4 时均超过 50ms 阈值(TP=8: 55ms+, TP=16: 52ms+) +- 这可能是 GLM-5.2 MoE 架构 + bf16 KV cache 的固有特性,需要 fp8 KV cache 或其他优化 diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env index d8b3a32..3db87cd 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env @@ -18,16 +18,16 @@ SERVED_MODEL_NAME="glm-5.2" VLLM_PORT="${VLLM_PORT:-30050}" # Dedicated container name so this experiment never touches other 910c runs. -CONTAINER_NAME="${CONTAINER_NAME:-vllm-ascend-glm52-910c}" +CONTAINER_NAME="vllm-ascend-glm52-910c" # Python interpreter for the benchmark client inside the vllm-ascend container. -CONTAINER_PYTHON="${CONTAINER_PYTHON:-/usr/local/bin/python}" +CONTAINER_PYTHON="/usr/local/python3.12.13/bin/python3" # vllm-ascend image. Override with the exact tag after `docker load`-ing one of: # /mnt/models/vllm-ascend-glm5.2-a3-openeuler.tar (GLM5.2-tuned, recommended) # /mnt/models/vllm-ascend-v0.23.0rc1-a3-openeuler.tar (general v0.23) USE_DOCKER="${USE_DOCKER:-1}" -DOCKER_IMAGE="${DOCKER_IMAGE:-vllm-ascend:glm5.2-a3-openeuler}" +DOCKER_IMAGE="${DOCKER_IMAGE:-local/vllm-ascend:0.23-a3-20260718-sglang}" # Benchmark client Docker image. vLLM's image does not include sglang.bench_serving; # reuse the vllm-ascend container itself for the client via `docker exec` (see @@ -69,7 +69,7 @@ fi # - block-size 128 matches Ascend page semantics (P99 of H20 uses 256; 910C favors 128). # - MAX_MODEL_LEN: GLM-5.2 supports up to 128K context; cap at 131072. # - gpu-memory-utilization maps to NPU HBM fraction on vllm-ascend (0.9 mirrors H20). -GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.95}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" BLOCK_SIZE="${BLOCK_SIZE:-128}" MAX_MODEL_LEN="${MAX_MODEL_LEN:-131072}" diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index e5d87eb..0cec9d8 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -97,6 +97,8 @@ engine_start_server() { local tp="$1" local dp="$2" local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + export ENGINE_TP="$tp" + export ENGINE_DP="$dp" log "starting vllm-ascend server tp=${tp} dp=${dp}" bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 if ! engine_is_healthy; then @@ -130,7 +132,7 @@ engine_run_bench() { local concurrency="$3" local num_prompts="$4" local output_file="$5" - local container_name="${CONTAINER_NAME}_tp${tp}_dp${dp}" + local container_name="${CONTAINER_NAME}_tp${ENGINE_TP:-${tp}}_dp${ENGINE_DP:-${dp}}" local container_output="/tmp/bench_outputs/adaptive_$(basename "$output_file")" if [[ "$output_file" == "/dev/null" ]]; then container_output="/dev/null" @@ -143,7 +145,7 @@ engine_run_bench() { --backend vllm --host 127.0.0.1 --port "$ENGINE_PORT" - --model "$SERVED_MODEL_NAME" + --tokenizer "$MODEL_PATH" --dataset-name "$BENCH_DATASET_NAME" --random-input-len "$isl" --random-output-len "$osl" @@ -170,7 +172,7 @@ engine_run_bench() { # host port. See envs/ASCEND_910C_ENV_SETUP.md. docker exec "$container_name" mkdir -p /tmp/bench_outputs 2>/dev/null || true docker exec "$container_name" \ - env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 \ + env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 HF_DATASETS_OFFLINE=1 TORCH_DEVICE_BACKEND_AUTOLOAD=0 \ "$CONTAINER_PYTHON" -m sglang.bench_serving "${bench_args[@]}" || return $? if [[ "$output_file" != "/dev/null" ]]; then @@ -179,7 +181,7 @@ engine_run_bench() { } export -f engine_run_bench -export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME +export ENGINE_PORT CONTAINER_NAME CONTAINER_PYTHON MODEL_PATH RESULT_BASE SERVED_MODEL_NAME ENGINE_TP ENGINE_DP export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON export SEARCH_START_CONCURRENCY=16 diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh index 2553617..2539627 100755 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -19,8 +19,7 @@ mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" IMAGE="${DOCKER_IMAGE:-vllm-ascend:glm5.2-a3-openeuler}" PORT="${VLLM_PORT:-30050}" -NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm_tp${TP}_dp${DP}}" -NAME="${NAME}_tp${TP}_dp${DP}" +NAME="${CONTAINER_NAME:-${EXPERIMENT}_vllm}_tp${TP}_dp${DP}" PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" @@ -30,8 +29,8 @@ rm -f "$PID_FILE" docker rm -f "$NAME" >/dev/null 2>&1 || true # vLLM-Ascend launch args. Differences vs NVIDIA vLLM: -# - no --no-enable-flashinfer-autotune (Ascend uses its own attention path) -# - --kv-cache-dtype may need to be fp16 if the image rejects fp8 on 910C +#\n - no --no-enable-flashinfer-autotune (Ascend uses its own attention path) +#\n - --kv-cache-dtype may need to be fp16 if the image rejects fp8 on 910C SERVER_ARGS=( "$MODEL_PATH" --served-model-name "$SERVED_MODEL_NAME" @@ -39,11 +38,14 @@ SERVER_ARGS=( --kv-cache-dtype "$KV_CACHE_DTYPE" --block-size "$BLOCK_SIZE" --tensor-parallel-size "$TP" + --enable-expert-parallel --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" --max-model-len "$MAX_MODEL_LEN" --max-num-seqs "$MAX_NUM_SEQS" --host 0.0.0.0 --port "$PORT" + --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' + --additional-config '{"enable_dsa_cp": true,"enable_sparse_sfa_c8": false, "enable_sparse_li_c8": true,"enable_balance_scheduling": true,"multistream_overlap_shared_expert":true}' ) if [[ "$DP" -gt 1 ]]; then @@ -65,16 +67,28 @@ echo "Log: $LOG" # adaptive search stop the container by killing the process (the container has # --rm so it self-cleans). The Ascend Docker Runtime is the default runtime, so # no --runtime flag is required. -nohup docker run --rm \ +nohup docker run --rm \ --name "$NAME" \ --ipc host \ - --shm-size 16g \ --network host \ --ulimit memlock=-1 \ + --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci2 --device /dev/davinci3 \ + --device /dev/davinci4 --device /dev/davinci5 --device /dev/davinci6 --device /dev/davinci7 \ + --device /dev/davinci8 --device /dev/davinci9 --device /dev/davinci10 --device /dev/davinci11 \ + --device /dev/davinci12 --device /dev/davinci13 --device /dev/davinci14 --device /dev/davinci15 \ + --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc \ + -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /usr/local/Ascend/driver/lib64:/usr/local/Ascend/driver/lib64 \ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \ + -v /etc/ascend_install.info:/etc/ascend_install.info \ + -v /etc/hccn.conf:/etc/hccn.conf \ -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ -v "${RUNTIME_BASE}/tmp:/tmp" \ - -e ASCEND_VISIBLE_DEVICES="${ASCEND_VISIBLE_DEVICES}" \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /mnt:/mnt \ -e PYTORCH_NPU_ALLOC_CONF=expandable_segments:True \ + -e VLLM_ASCEND_ENABLE_FLASHCOMM1=1 \ -e PYTHONUNBUFFERED=1 \ -e HF_HUB_OFFLINE=1 \ -e TRANSFORMERS_OFFLINE=1 \ @@ -87,21 +101,32 @@ echo $PID > "$PID_FILE" echo "PID: $PID" echo "Waiting for health on port ${PORT}..." -for i in $(seq 1 240); do +# Health-check budget. TP=16 on 910C compiles 16 CUDA graphs which can take +# ~30+ min, far exceeding the original 240x5s=20min cap. Allow overrides. +HEALTH_MAX_RETRIES="${HEALTH_MAX_RETRIES:-480}" +HEALTH_RETRY_INTERVAL_S="${HEALTH_RETRY_INTERVAL_S:-5}" + +for i in $(seq 1 "$HEALTH_MAX_RETRIES"); do if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then - echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}" - echo "Log: $LOG" - exit 0 + + echo "vLLM-Ascend server is ready at http://127.0.0.1:${PORT}" + + echo "Log: $LOG" + + exit 0 fi if ! kill -0 $PID 2>/dev/null; then - echo "ERROR: Docker vLLM-Ascend server exited early" - tail -200 "$LOG" - exit 1 + + echo "ERROR: Docker vLLM-Ascend server exited early" + + tail -200 "$LOG" + + exit 1 fi - echo "Waiting... ($i/240)" - sleep 5 + echo "Waiting... ($i/$HEALTH_MAX_RETRIES)" + sleep "$HEALTH_RETRY_INTERVAL_S" done -echo "ERROR: vLLM-Ascend server not healthy after 240 retries" +echo "ERROR: vLLM-Ascend server not healthy after $HEALTH_MAX_RETRIES retries" tail -200 "$LOG" exit 1 diff --git a/scripts/common/adaptive_concurrency.py b/scripts/common/adaptive_concurrency.py index e8de21a..2d025c5 100755 --- a/scripts/common/adaptive_concurrency.py +++ b/scripts/common/adaptive_concurrency.py @@ -99,12 +99,71 @@ def last_json_object(path: Path) -> dict[str, Any]: return rows[-1] +def _percentile(values, pct): + """Compute a percentile (0-100) from a list of float values in *seconds*; + return value in *ms*.""" + if not values: + return 0.0 + s = sorted(values) + if len(s) == 1: + return s[0] * 1000.0 + if pct <= 0: + return s[0] * 1000.0 + if pct >= 100: + return s[-1] * 1000.0 + rank = (pct / 100.0) * (len(s) - 1) + lo = int(rank) + hi = min(lo + 1, len(s) - 1) + frac = rank - lo + return (s[lo] * (1.0 - frac) + s[hi] * frac) * 1000.0 + + def percentile_value(data: dict[str, Any], prefix: str, percentile: str) -> float: + """Read a pre-computed percentile from the raw bench output. + + sglang 0.5.x omits p95 (and sometimes p50) for ttft/tpot/e2e, only + emitting p99 plus p95_itl_ms. When the field is missing, recompute it + from the per-request arrays (ttfts/itls) that sglang always emits, so + the adaptive search and reports stay populated. + """ if percentile == "p50": key = f"median_{prefix}_ms" else: key = f"{percentile}_{prefix}_ms" - return float(data.get(key, 0.0) or 0.0) + val = data.get(key) + if val is not None: + return float(val) + + pct_num = {"p95": 95.0, "p99": 99.0, "p50": 50.0}.get(percentile, 0.0) + if prefix == "ttft": + arr = [float(v) for v in data.get("ttfts", []) if v is not None] + return _percentile(arr, pct_num) + if prefix == "itl": + flat = [] + for sub in data.get("itls", []): + if isinstance(sub, list): + flat.extend(float(v) for v in sub if v is not None) + elif sub is not None: + flat.append(float(sub)) + return _percentile(flat, pct_num) + if prefix == "tpot": + per_req = [] + for sub in data.get("itls", []): + if isinstance(sub, list) and sub: + per_req.append(sum(float(v) for v in sub if v is not None) / len(sub)) + return _percentile(per_req, pct_num) + if prefix == "e2e_latency": + ttfts = [float(v) for v in data.get("ttfts", []) if v is not None] + per_req = [] + itls = data.get("itls", []) + for i, ttft in enumerate(ttfts): + tail = itls[i] if i < len(itls) else [] + if isinstance(tail, list): + per_req.append(ttft + sum(float(v) for v in tail if v is not None)) + else: + per_req.append(ttft) + return _percentile(per_req, pct_num) + return 0.0 def command_shapes(args: argparse.Namespace) -> int: @@ -191,7 +250,12 @@ def command_parse_result(args: argparse.Namespace) -> int: "request_tps": float(data.get("request_throughput", 0.0) or 0.0), "input_tps": float(data.get("input_throughput", 0.0) or 0.0), "output_tps": float(data.get("output_throughput", 0.0) or 0.0), - "total_tps": float(data.get("total_throughput", 0.0) or 0.0), + "total_tps": float( + data.get("total_throughput") + or (float(data.get("input_throughput", 0.0) or 0.0) + + float(data.get("output_throughput", 0.0) or 0.0)) + or 0.0 + ), "mean_input_tokens": mean_input, "mean_output_tokens": mean_output, "ttft_p50_ms": percentile_value(data, "ttft", "p50"), @@ -223,7 +287,9 @@ def command_gain(args: argparse.Namespace) -> int: else: gain_pct = (args.current - args.previous) / args.previous * 100.0 meaningful = gain_pct >= args.threshold_pct - gain_text = "inf" if gain_pct == float("inf") else f"{gain_pct:.6f}" + # Emit Infinity (JSON-valid) rather than inf so downstream json.loads() + # in adaptive_append_point_from_metrics does not crash. + gain_text = "Infinity" if gain_pct == float("inf") else f"{gain_pct:.6f}" print(f"{gain_text}\t{1 if meaningful else 0}") return 0 diff --git a/scripts/common/parse_backend.py b/scripts/common/parse_backend.py index 7414a9d..4ee4b21 100755 --- a/scripts/common/parse_backend.py +++ b/scripts/common/parse_backend.py @@ -7,6 +7,8 @@ and updates results.json + report.md in the given result root. Usage: python3 parse_backend.py [--backend sglang|vllm] """ +from __future__ import annotations + import argparse import json from pathlib import Path @@ -25,6 +27,72 @@ def parse_jsonl(path: Path) -> dict | None: return None +def _percentile(values, pct): + """Compute a percentile (0-100) from a list of float values in *seconds*; + return value in *ms*.""" + if not values: + return 0.0 + s = sorted(values) + if len(s) == 1: + return s[0] * 1000.0 + if pct <= 0: + return s[0] * 1000.0 + if pct >= 100: + return s[-1] * 1000.0 + rank = (pct / 100.0) * (len(s) - 1) + lo = int(rank) + hi = min(lo + 1, len(s) - 1) + frac = rank - lo + return (s[lo] * (1.0 - frac) + s[hi] * frac) * 1000.0 + + +def _pct_fallback(data, prefix, percentile): + """Recompute a percentile from per-request arrays when the engine + (sglang 0.5.x) omits the pre-computed p95/p50 field.""" + pct_num = {"p95": 95.0, "p99": 99.0, "p50": 50.0}.get(percentile, 0.0) + if prefix == "ttft": + arr = [float(v) for v in data.get("ttfts", []) if v is not None] + return _percentile(arr, pct_num) + if prefix == "itl": + flat = [] + for sub in data.get("itls", []): + if isinstance(sub, list): + flat.extend(float(v) for v in sub if v is not None) + elif sub is not None: + flat.append(float(sub)) + return _percentile(flat, pct_num) + if prefix == "tpot": + per_req = [] + for sub in data.get("itls", []): + if isinstance(sub, list) and sub: + per_req.append(sum(float(v) for v in sub if v is not None) / len(sub)) + return _percentile(per_req, pct_num) + if prefix == "e2e_latency": + ttfts = [float(v) for v in data.get("ttfts", []) if v is not None] + per_req = [] + itls = data.get("itls", []) + for i, ttft in enumerate(ttfts): + tail = itls[i] if i < len(itls) else [] + if isinstance(tail, list): + per_req.append(ttft + sum(float(v) for v in tail if v is not None)) + else: + per_req.append(ttft) + return _percentile(per_req, pct_num) + return 0.0 + + +def _pct(data, prefix, percentile): + """Read pre-computed percentile, fall back to per-request arrays.""" + if percentile == "p50": + key = f"median_{prefix}_ms" + else: + key = f"{percentile}_{prefix}_ms" + val = data.get(key) + if val is not None: + return float(val) + return _pct_fallback(data, prefix, percentile) + + def compute_metrics(data: dict) -> dict: completed = data.get("completed", 0) total = len(data.get("input_lens", [])) @@ -38,36 +106,41 @@ def compute_metrics(data: dict) -> dict: "request_throughput": data.get("request_throughput", 0.0), "input_token_throughput": data.get("input_throughput", 0.0), "output_token_throughput": data.get("output_throughput", 0.0), - "total_token_throughput": data.get("total_throughput", 0.0), + "total_token_throughput": ( + data.get("total_throughput") + if data.get("total_throughput") + else (float(data.get("input_throughput", 0.0) or 0.0) + + float(data.get("output_throughput", 0.0) or 0.0)) + ), "total_input_tokens": data.get("total_input_tokens", 0), "total_output_tokens": data.get("total_output_tokens", 0), "e2e_ms": { "mean": data.get("mean_e2e_latency_ms", 0.0), "p50": data.get("median_e2e_latency_ms", 0.0), "p90": data.get("p90_e2e_latency_ms", 0.0), - "p95": data.get("p95_e2e_latency_ms", 0.0), - "p99": data.get("p99_e2e_latency_ms", 0.0), + "p95": _pct(data, "e2e_latency", "p95"), + "p99": _pct(data, "e2e_latency", "p99"), }, "ttft_ms": { "mean": data.get("mean_ttft_ms", 0.0), "p50": data.get("median_ttft_ms", 0.0), "p90": data.get("p90_ttft_ms", 0.0), - "p95": data.get("p95_ttft_ms", 0.0), - "p99": data.get("p99_ttft_ms", 0.0), + "p95": _pct(data, "ttft", "p95"), + "p99": _pct(data, "ttft", "p99"), }, "tpot_ms": { "mean": data.get("mean_tpot_ms", 0.0), "p50": data.get("median_tpot_ms", 0.0), "p90": data.get("p90_tpot_ms", 0.0), - "p95": data.get("p95_tpot_ms", 0.0), - "p99": data.get("p99_tpot_ms", 0.0), + "p95": _pct(data, "tpot", "p95"), + "p99": _pct(data, "tpot", "p99"), }, "itl_ms": { "mean": data.get("mean_itl_ms", 0.0), "p50": data.get("median_itl_ms", 0.0), "p90": data.get("p90_itl_ms", 0.0), - "p95": data.get("p95_itl_ms", 0.0), - "p99": data.get("p99_itl_ms", 0.0), + "p95": _pct(data, "itl", "p95"), + "p99": _pct(data, "itl", "p99"), }, }