add dsv4_h200_vllm_tp_dp_matrix experiment (TP×DP matrix with Y-only mode, 1M excluded)
This commit is contained in:
yy-fighting
parent
1482601ae5
commit
04a384b265
162
experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py
Executable file
162
experiments/dsv4_h200_vllm_tp_dp_matrix/compare.py
Executable file
@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross TP×DP configuration comparison for dsv4_h200_vllm_tp_dp_matrix.
|
||||
|
||||
Usage:
|
||||
python3 compare.py --run-root results/<run_id> [--output comparison.md]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_result(result_root: Path) -> dict:
|
||||
path = result_root / "results.json"
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
|
||||
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str:
|
||||
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
||||
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
||||
if ttft_ok and tpot_ok:
|
||||
return "✅"
|
||||
if ttft_ok or tpot_ok:
|
||||
return "⚠️"
|
||||
return "❌"
|
||||
|
||||
|
||||
def gpu_memory_str(gpu: dict | None) -> str:
|
||||
if not gpu:
|
||||
return "-"
|
||||
peak = gpu.get("peak_used_mb", 0)
|
||||
total = gpu.get("memory_total_mb", 0)
|
||||
if total:
|
||||
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
|
||||
return f"{peak:.0f}"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-root", type=Path, required=True)
|
||||
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
||||
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Discover configurations: tp*_dp* directories.
|
||||
configs = []
|
||||
for subdir in sorted(args.run_root.iterdir()):
|
||||
if not subdir.is_dir():
|
||||
continue
|
||||
name = subdir.name
|
||||
if not (name.startswith("tp") and "_dp" in name):
|
||||
continue
|
||||
results_json = subdir / "results.json"
|
||||
if not results_json.exists():
|
||||
continue
|
||||
configs.append((name, load_result(subdir)))
|
||||
|
||||
if not configs:
|
||||
print(f"No tp*_dp* results found under {args.run_root}")
|
||||
return
|
||||
|
||||
model = configs[0][1].get("metadata", {}).get("model", "unknown")
|
||||
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
|
||||
|
||||
# Group by scenario name.
|
||||
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
|
||||
skipped: dict[str, dict[str, str]] = defaultdict(dict)
|
||||
for label, data in configs:
|
||||
for s in data.get("scenarios", []):
|
||||
key = s["name"]
|
||||
if s.get("status") == "skipped_oom":
|
||||
skipped[key][label] = s.get("note", "skipped")
|
||||
else:
|
||||
by_scenario[key][label] = s
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n")
|
||||
f.write("## Summary\n\n")
|
||||
f.write(f"- Model: `{model}`\n")
|
||||
f.write(f"- Hardware: {hardware}\n")
|
||||
f.write("- Backend: vLLM\n")
|
||||
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
||||
|
||||
# Configuration overview.
|
||||
f.write("### Configurations\n\n")
|
||||
f.write("| Config | TP | DP | GPUs/replica | Notes |\n")
|
||||
f.write("|---|---:|---:|---:|---|\n")
|
||||
for label, data in configs:
|
||||
cfg = data.get("config", {})
|
||||
tp = cfg.get("tp", "?")
|
||||
dp = cfg.get("dp", "?")
|
||||
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
|
||||
f.write("\n")
|
||||
|
||||
# Side-by-side table.
|
||||
f.write("## Side-by-side results\n\n")
|
||||
headers = [
|
||||
"Scenario", "ISL", "DSL", "Config", "Conc", "Req/s", "OutTok/s",
|
||||
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
|
||||
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
|
||||
]
|
||||
f.write("| " + " | ".join(headers) + " |\n")
|
||||
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
|
||||
|
||||
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, x.replace("c", "").replace("i", "_").replace("o", "_").split("_")[1:]))):
|
||||
cfg_part, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split()
|
||||
# cfg_part not used; just for readability.
|
||||
for label, data in configs:
|
||||
s = by_scenario[scenario_name].get(label)
|
||||
if s is None:
|
||||
if scenario_name in skipped and label in skipped[scenario_name]:
|
||||
note = skipped[scenario_name][label]
|
||||
f.write(f"| {scenario_name} | {isl} | {dsl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
|
||||
continue
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||
gpu = m.get("gpu_memory")
|
||||
f.write(
|
||||
f"| {scenario_name} | {isl} | {dsl} | {label} | {cfg['concurrency']} | "
|
||||
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
|
||||
)
|
||||
|
||||
# Best throughput per ISL/DSL.
|
||||
f.write("\n## Best throughput per (ISL, DSL)\n\n")
|
||||
f.write("| ISL | DSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
|
||||
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
|
||||
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
|
||||
for scenario_name, backends in by_scenario.items():
|
||||
_, isl, dsl = scenario_name.replace("c", " ").replace("i", " ").replace("o", " ").split()
|
||||
isl_i, dsl_i = int(isl), int(dsl)
|
||||
for label, s in backends.items():
|
||||
m = s["metrics"]
|
||||
out_tok = m["output_token_throughput"]
|
||||
if (isl_i, dsl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, dsl_i)][0]:
|
||||
best_by_shape[(isl_i, dsl_i)] = (out_tok, label, s)
|
||||
for (isl_i, dsl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
|
||||
m = s["metrics"]
|
||||
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||
f.write(
|
||||
f"| {isl_i} | {dsl_i} | {label} | {s['config']['concurrency']} | "
|
||||
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
|
||||
)
|
||||
|
||||
f.write("\n## Notes\n\n")
|
||||
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
||||
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
|
||||
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
|
||||
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
|
||||
|
||||
print(f"Wrote comparison to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
43
experiments/dsv4_h200_vllm_tp_dp_matrix/config.env
Normal file
43
experiments/dsv4_h200_vllm_tp_dp_matrix/config.env
Normal file
@ -0,0 +1,43 @@
|
||||
# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs).
|
||||
# Tests vLLM with three parallel configurations:
|
||||
# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas
|
||||
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
||||
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
||||
|
||||
EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix"
|
||||
MODEL_NAME="DeepSeek-V4-Flash"
|
||||
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
||||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||||
|
||||
VLLM_PORT="${VLLM_PORT:-30030}"
|
||||
# vLLM 0.24.0 multi-port DP supervisor listens on this hard-coded port.
|
||||
VLLM_DP_SUPERVISOR_PORT="${VLLM_DP_SUPERVISOR_PORT:-9256}"
|
||||
|
||||
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
|
||||
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
|
||||
# The benchmark client is sglang.bench_serving, even when the backend is vLLM.
|
||||
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
|
||||
|
||||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||
|
||||
# Parallel configurations to test. Format: "TP DP"
|
||||
declare -a PARALLEL_CONFIGS=(
|
||||
"2 4"
|
||||
"4 2"
|
||||
"8 1"
|
||||
)
|
||||
|
||||
# Matrix and concurrency rules are defined in matrix.json.
|
||||
# This file is consumed by generate_scenarios.py.
|
||||
MATRIX_FILE="${SCRIPT_DIR:-.}/matrix.json"
|
||||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||
|
||||
# Sampling density for concurrency. 0 means use the default heuristic in
|
||||
# generate_scenarios.py (6-8 points, or all integers when range is small).
|
||||
CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-0}"
|
||||
|
||||
# Per-scenario timeout to avoid hangs (seconds).
|
||||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||
|
||||
# GPU memory sampling interval (seconds).
|
||||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
105
experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py
Executable file
105
experiments/dsv4_h200_vllm_tp_dp_matrix/generate_scenarios.py
Executable file
@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the scenario list for the TP×DP matrix experiment.
|
||||
|
||||
Reads matrix.json and prints TSV lines:
|
||||
|
||||
mark input_len output_len concurrency num_prompts
|
||||
|
||||
mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sample_concurrency(low: int, high: int, target: int) -> list[int]:
|
||||
"""Return a sorted list of concurrency values in [low, high].
|
||||
|
||||
If the range is small, return every integer. Otherwise sample roughly
|
||||
`target` points linearly between low and high (inclusive).
|
||||
"""
|
||||
assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
|
||||
if high - low + 1 <= target:
|
||||
return list(range(low, high + 1))
|
||||
|
||||
points = set()
|
||||
points.add(low)
|
||||
points.add(high)
|
||||
step = (high - low) / (target - 1)
|
||||
for i in range(1, target - 1):
|
||||
v = low + round(step * i)
|
||||
points.add(max(low, min(high, v)))
|
||||
return sorted(points)
|
||||
|
||||
|
||||
def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
|
||||
with open(matrix_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
matrix = data["matrix"]
|
||||
concurrency_cfg = data["concurrency"]
|
||||
|
||||
scenarios = []
|
||||
for isl_str in sorted(matrix.keys(), key=int):
|
||||
dsl_map = matrix[isl_str]
|
||||
low = concurrency_cfg[isl_str]["low"]
|
||||
high = concurrency_cfg[isl_str]["high"]
|
||||
concurrencies = sample_concurrency(low, high, target_samples)
|
||||
|
||||
for dsl_str in sorted(dsl_map.keys(), key=int):
|
||||
mark = dsl_map[dsl_str]
|
||||
if mode == "Y" and mark != "Y":
|
||||
continue
|
||||
if mode == "Y+P" and mark not in ("Y", "P"):
|
||||
continue
|
||||
# mode == "all" keeps everything, including N.
|
||||
|
||||
for conc in concurrencies:
|
||||
scenarios.append(
|
||||
{
|
||||
"mark": mark,
|
||||
"input_len": int(isl_str),
|
||||
"output_len": int(dsl_str),
|
||||
"concurrency": conc,
|
||||
"num_prompts": conc * 5,
|
||||
}
|
||||
)
|
||||
|
||||
return scenarios
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--matrix", type=Path, default=Path("matrix.json"))
|
||||
parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None,
|
||||
help="Scenario selection mode. Defaults to matrix.mode.")
|
||||
parser.add_argument("--target-samples", type=int, default=0,
|
||||
help="Target number of concurrency samples. 0 = heuristic (6-8).")
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.matrix, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
mode = args.mode if args.mode else data.get("mode", "Y+P")
|
||||
|
||||
target_samples = args.target_samples
|
||||
if target_samples <= 0:
|
||||
env_samples = os.getenv("CONCURRENCY_SAMPLES", "0")
|
||||
try:
|
||||
target_samples = int(env_samples)
|
||||
except ValueError:
|
||||
target_samples = 0
|
||||
if target_samples <= 0:
|
||||
target_samples = 7
|
||||
|
||||
scenarios = generate_scenarios(args.matrix, mode, target_samples)
|
||||
|
||||
print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts")
|
||||
for s in scenarios:
|
||||
print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json
Normal file
81
experiments/dsv4_h200_vllm_tp_dp_matrix/matrix.json
Normal file
@ -0,0 +1,81 @@
|
||||
{
|
||||
"comment": "ISL/DSL matrix for dsv4_h200_vllm_tp_dp_matrix. Y=must test, P=optional (record N on failure), N=skip.",
|
||||
"mode": "Y",
|
||||
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
|
||||
"matrix": {
|
||||
"1024": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "Y"
|
||||
},
|
||||
"4096": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "Y"
|
||||
},
|
||||
"16384": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "Y",
|
||||
"4096": "P"
|
||||
},
|
||||
"65536": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "Y",
|
||||
"2048": "P",
|
||||
"4096": "N"
|
||||
},
|
||||
"131072": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "Y",
|
||||
"1024": "P",
|
||||
"2048": "N",
|
||||
"4096": "N"
|
||||
},
|
||||
"262144": {
|
||||
"128": "Y",
|
||||
"256": "Y",
|
||||
"512": "P",
|
||||
"1024": "N",
|
||||
"2048": "N",
|
||||
"4096": "N"
|
||||
},
|
||||
"524288": {
|
||||
"128": "Y",
|
||||
"256": "P",
|
||||
"512": "N",
|
||||
"1024": "N",
|
||||
"2048": "N",
|
||||
"4096": "N"
|
||||
},
|
||||
"1048576": {
|
||||
"128": "N",
|
||||
"256": "N",
|
||||
"512": "N",
|
||||
"1024": "N",
|
||||
"2048": "N",
|
||||
"4096": "N"
|
||||
}
|
||||
},
|
||||
"concurrency": {
|
||||
"1024": { "low": 1, "high": 128 },
|
||||
"4096": { "low": 1, "high": 64 },
|
||||
"16384": { "low": 1, "high": 32 },
|
||||
"65536": { "low": 1, "high": 8 },
|
||||
"131072": { "low": 1, "high": 4 },
|
||||
"262144": { "low": 1, "high": 2 },
|
||||
"524288": { "low": 1, "high": 2 },
|
||||
"1048576": { "low": 1, "high": 2 }
|
||||
}
|
||||
}
|
||||
394
experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh
Executable file
394
experiments/dsv4_h200_vllm_tp_dp_matrix/run_bench.sh
Executable file
@ -0,0 +1,394 @@
|
||||
#!/usr/bin/env bash
|
||||
# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM.
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
|
||||
MATRIX_MODE="${MATRIX_MODE:-Y+P}"
|
||||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
|
||||
PYTHON="${VENV_CLIENT}/bin/python"
|
||||
|
||||
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
||||
mkdir -p "$log_dir_global"
|
||||
log_init "${log_dir_global}/orchestrator.log"
|
||||
|
||||
log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
|
||||
log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
health_port_for() {
|
||||
local dp="$1"
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
echo "$VLLM_DP_SUPERVISOR_PORT"
|
||||
else
|
||||
echo "$VLLM_PORT"
|
||||
fi
|
||||
}
|
||||
|
||||
is_server_healthy() {
|
||||
local port="$1"
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
stop_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local pid_file="/data/user1/yy/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||
|
||||
if [[ -f "$pid_file" ]]; then
|
||||
local pid
|
||||
pid="$(cat "$pid_file")"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})"
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$pid_file"
|
||||
fi
|
||||
|
||||
# Fallback: kill any vllm serve processes for this model.
|
||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
||||
pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
build_server_args() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local max_model_len="$3"
|
||||
local max_num_seqs="$4"
|
||||
|
||||
local args=(
|
||||
"vllm serve" "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--tensor-parallel-size "$tp"
|
||||
--kv-cache-dtype fp8
|
||||
--max-model-len "$max_model_len"
|
||||
--max-num-seqs "$max_num_seqs"
|
||||
--block-size 256
|
||||
--gpu-memory-utilization 0.90
|
||||
--tokenizer-mode deepseek_v4
|
||||
--reasoning-parser deepseek_v4
|
||||
--no-disable-hybrid-kv-cache-manager
|
||||
--disable-uvicorn-access-log
|
||||
--port "$VLLM_PORT"
|
||||
)
|
||||
if [[ "$dp" -gt 1 ]]; then
|
||||
args+=(
|
||||
--data-parallel-size "$dp"
|
||||
--data-parallel-size-local "$dp"
|
||||
--data-parallel-multi-port-external-lb
|
||||
)
|
||||
fi
|
||||
printf '%s ' "${args[@]}"
|
||||
}
|
||||
|
||||
start_server() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local max_model_len="$3"
|
||||
local max_num_seqs="$4"
|
||||
|
||||
log "starting vllm server tp=${tp} dp=${dp} max_model_len=${max_model_len} max_num_seqs=${max_num_seqs}"
|
||||
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" "$max_model_len" "$max_num_seqs" \
|
||||
>> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||
|
||||
local hport
|
||||
hport="$(health_port_for "$dp")"
|
||||
if ! is_server_healthy "$hport"; then
|
||||
log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${hport}"
|
||||
return 1
|
||||
fi
|
||||
log "vllm server tp=${tp} dp=${dp} is healthy on port ${hport}"
|
||||
}
|
||||
|
||||
run_warmup() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local input_len="$3"
|
||||
local output_len="$4"
|
||||
local hport
|
||||
hport="$(health_port_for "$dp")"
|
||||
|
||||
log "warming up tp=${tp} dp=${dp} (input=${input_len}, output=${output_len}, num=1)"
|
||||
"$PYTHON" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
|
||||
--backend vllm \
|
||||
--host 127.0.0.1 \
|
||||
--port "$hport" \
|
||||
--input-len "$input_len" \
|
||||
--output-len "$output_len" \
|
||||
--num 1 \
|
||||
--env-python "$PYTHON" \
|
||||
>> "${log_dir_global}/vllm_tp${tp}_dp${dp}.warmup.log" 2>&1
|
||||
log "warmup completed"
|
||||
}
|
||||
|
||||
scenario_already_completed() {
|
||||
local output_file="$1"
|
||||
local expected="$2"
|
||||
[[ -s "$output_file" ]] || return 1
|
||||
local completed
|
||||
completed="$("$PYTHON" -c "
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
print(data.get('completed', 0))
|
||||
break
|
||||
except Exception:
|
||||
print(0)
|
||||
" "$output_file")"
|
||||
[[ "${completed:-0}" -ge "$expected" ]]
|
||||
}
|
||||
|
||||
start_gpu_monitor() {
|
||||
local csv_path="$1"
|
||||
mkdir -p "$(dirname "$csv_path")"
|
||||
nvidia-smi \
|
||||
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \
|
||||
--format=csv \
|
||||
-l "$GPU_MEM_SAMPLE_INTERVAL_S" \
|
||||
> "$csv_path" 2>/dev/null &
|
||||
echo $!
|
||||
}
|
||||
|
||||
stop_gpu_monitor() {
|
||||
local pid="$1"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
append_scenario_record() {
|
||||
local result_root="$1"
|
||||
local json_path="$result_root/results.json"
|
||||
shift
|
||||
# Remaining args are key=value pairs.
|
||||
local scenario_json
|
||||
scenario_json="$("$PYTHON" -c "
|
||||
import json, sys
|
||||
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||
d = {k: json.loads(v) for k, v in pairs}
|
||||
print(json.dumps(d, ensure_ascii=False))
|
||||
" "$@")"
|
||||
PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-configuration runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
run_parallel_config() {
|
||||
local tp="$1"
|
||||
local dp="$2"
|
||||
local config_label="tp${tp}_dp${dp}"
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}"
|
||||
local raw_dir="${result_root}/raw_outputs"
|
||||
local gpu_log_dir="${result_root}/gpu_logs"
|
||||
local phase_log_dir="${result_root}/logs"
|
||||
mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir"
|
||||
|
||||
log "===== ${config_label} START ====="
|
||||
|
||||
# Generate scenario list for this config.
|
||||
local scenario_tsv="${result_root}/scenarios.tsv"
|
||||
"$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \
|
||||
--matrix "$MATRIX_FILE" \
|
||||
--mode "$MATRIX_MODE" \
|
||||
> "$scenario_tsv"
|
||||
local total_scenarios
|
||||
total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
|
||||
log "generated ${total_scenarios} scenarios for ${config_label}"
|
||||
|
||||
if [[ "$total_scenarios" -eq 0 ]]; then
|
||||
log "no scenarios for ${config_label}; skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Write metadata.
|
||||
ensure_result_root "$result_root"
|
||||
write_metadata_json \
|
||||
"${result_root}/results.json" \
|
||||
"${EXPERIMENT_NAME}_${config_label}" \
|
||||
"$RUN_ID" \
|
||||
"$MODEL_PATH" \
|
||||
"vllm" \
|
||||
"vllm" \
|
||||
"$HARDWARE" \
|
||||
"$ACCELERATOR" \
|
||||
"$CHIP" \
|
||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||
"$VENV_VLLM" \
|
||||
"H200 vLLM TP×DP matrix for DeepSeek-V4-Flash"
|
||||
|
||||
# Embed static config now; server_args will be updated per ISL group.
|
||||
jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" \
|
||||
'.config = {
|
||||
"tp": ($tp | tonumber),
|
||||
"dp": ($dp | tonumber),
|
||||
"cuda_visible_devices": $cuda,
|
||||
"backend": "vllm",
|
||||
"server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh"
|
||||
}' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \
|
||||
mv "${result_root}/results.json.tmp" "${result_root}/results.json"
|
||||
|
||||
# Group scenarios by input_len. We start one server per input_len with a
|
||||
# max_model_len large enough for the biggest output_len in that group.
|
||||
local current_isl=""
|
||||
local max_dsl_for_isl=0
|
||||
local max_conc_for_isl=0
|
||||
local group_started=false
|
||||
|
||||
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl dsl conc num; do
|
||||
# When the input length changes, restart the server for the new group.
|
||||
if [[ "$isl" != "$current_isl" ]]; then
|
||||
if [[ "$group_started" == true ]]; then
|
||||
stop_server "$tp" "$dp"
|
||||
fi
|
||||
current_isl="$isl"
|
||||
max_dsl_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($3>max) max=$3} END{print max+0}' "$scenario_tsv")"
|
||||
max_conc_for_isl="$(awk -F'\t' -v isl="$isl" '$2==isl {if($4>max) max=$4} END{print max+0}' "$scenario_tsv")"
|
||||
|
||||
local max_model_len=$((isl + max_dsl_for_isl + 64))
|
||||
local max_num_seqs=$((max_conc_for_isl + 8))
|
||||
|
||||
stop_server "$tp" "$dp"
|
||||
if ! start_server "$tp" "$dp" "$max_model_len" "$max_num_seqs"; then
|
||||
log "ERROR: ${config_label} failed to start for ISL=${isl}; skipping this group"
|
||||
group_started=false
|
||||
continue
|
||||
fi
|
||||
group_started=true
|
||||
|
||||
# Update metadata with the actual server args for this ISL group.
|
||||
local server_args_str
|
||||
server_args_str="$(build_server_args "$tp" "$dp" "$max_model_len" "$max_num_seqs")"
|
||||
jq --arg isl "$isl" --arg args "$server_args_str" \
|
||||
'.config.server_args_per_isl += {($isl): $args}' \
|
||||
"${result_root}/results.json" > "${result_root}/results.json.tmp" && \
|
||||
mv "${result_root}/results.json.tmp" "${result_root}/results.json"
|
||||
|
||||
# Warmup with the shortest output for this ISL.
|
||||
run_warmup "$tp" "$dp" "$isl" 128
|
||||
fi
|
||||
|
||||
# If the server for this group did not start, skip all scenarios in it.
|
||||
if [[ "$group_started" != true ]]; then
|
||||
log "skipping ${config_label} ISL=${isl} scenario (server not started)"
|
||||
continue
|
||||
fi
|
||||
|
||||
local output_file="${raw_dir}/vllm_main_c${conc}_i${isl}_o${dsl}.jsonl"
|
||||
local detail_log="${phase_log_dir}/vllm_${config_label}_c${conc}_i${isl}_o${dsl}.log"
|
||||
local gpu_csv="${gpu_log_dir}/gpu_mem_c${conc}_i${isl}_o${dsl}.csv"
|
||||
|
||||
if scenario_already_completed "$output_file" "$num"; then
|
||||
log "skipping already-completed ${config_label} scenario: c=${conc} i=${isl} o=${dsl}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${dsl} n=${num}"
|
||||
|
||||
local gpu_pid
|
||||
gpu_pid="$(start_gpu_monitor "$gpu_csv")"
|
||||
|
||||
local bench_rc=0
|
||||
timeout "$SCENARIO_TIMEOUT_S" \
|
||||
"$PYTHON" -m sglang.bench_serving \
|
||||
--backend vllm \
|
||||
--host 127.0.0.1 \
|
||||
--port "$(health_port_for "$dp")" \
|
||||
--dataset-name random \
|
||||
--random-input-len "$isl" \
|
||||
--random-output-len "$dsl" \
|
||||
--num-prompts "$num" \
|
||||
--max-concurrency "$conc" \
|
||||
--request-rate 10000 \
|
||||
--output-file "$output_file" \
|
||||
--output-details \
|
||||
> "$detail_log" 2>&1 || bench_rc=$?
|
||||
|
||||
stop_gpu_monitor "$gpu_pid"
|
||||
|
||||
if [[ "$bench_rc" -ne 0 ]]; then
|
||||
log "ERROR: ${config_label} scenario c=${conc} i=${isl} o=${dsl} failed (rc=${bench_rc}); see ${detail_log}"
|
||||
if [[ "$mark" == "P" ]]; then
|
||||
log "optional (P) scenario failed; recording as skipped_oom and continuing"
|
||||
append_scenario_record "$result_root" \
|
||||
"name=c${conc}_i${isl}_o${dsl}" \
|
||||
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$dsl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||
"status=\"skipped_oom\"" \
|
||||
"note=\"bench command failed or timed out (rc=${bench_rc})\""
|
||||
continue
|
||||
else
|
||||
log "mandatory (Y) scenario failed; aborting current ISL group"
|
||||
stop_server "$tp" "$dp"
|
||||
group_started=false
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
log "finished ${config_label} scenario: output=${output_file}"
|
||||
done
|
||||
|
||||
stop_server "$tp" "$dp"
|
||||
|
||||
# Parse results.
|
||||
log "parsing ${config_label} results"
|
||||
"$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \
|
||||
>> "${phase_log_dir}/parse.log" 2>&1 || {
|
||||
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
|
||||
}
|
||||
|
||||
log "===== ${config_label} DONE ====="
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cleanup any leftovers.
|
||||
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||
read -r tp dp <<< "$cfg"
|
||||
stop_server "$tp" "$dp"
|
||||
done
|
||||
|
||||
# Run each parallel configuration.
|
||||
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||
read -r tp dp <<< "$cfg"
|
||||
run_parallel_config "$tp" "$dp"
|
||||
done
|
||||
|
||||
# Generate cross-configuration comparison.
|
||||
log "generating comparison report"
|
||||
"$PYTHON" "${SCRIPT_DIR}/compare.py" \
|
||||
--run-root "${RESULT_BASE}/${RUN_ID}" \
|
||||
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
||||
>> "${log_dir_global}/compare.log" 2>&1 || {
|
||||
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
||||
}
|
||||
|
||||
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|
||||
18
experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json
Normal file
18
experiments/dsv4_h200_vllm_tp_dp_matrix/smoke_matrix.json
Normal file
@ -0,0 +1,18 @@
|
||||
{
|
||||
"comment": "Small smoke matrix for quickly verifying TP×DP server startup and bench path.",
|
||||
"mode": "Y",
|
||||
"matrix": {
|
||||
"1024": {
|
||||
"128": "Y",
|
||||
"256": "Y"
|
||||
},
|
||||
"4096": {
|
||||
"128": "Y",
|
||||
"256": "Y"
|
||||
}
|
||||
},
|
||||
"concurrency": {
|
||||
"1024": { "low": 1, "high": 8 },
|
||||
"4096": { "low": 1, "high": 4 }
|
||||
}
|
||||
}
|
||||
92
experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh
Executable file
92
experiments/dsv4_h200_vllm_tp_dp_matrix/start_vllm_dp.sh
Executable file
@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start vLLM server for a given TP×DP configuration.
|
||||
# Usage: start_vllm_dp.sh <TP> <DP> <max_model_len> <max_num_seqs>
|
||||
set -e
|
||||
|
||||
TP="${1}"
|
||||
DP="${2}"
|
||||
MAX_MODEL_LEN="${3}"
|
||||
MAX_NUM_SEQS="${4}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
cd /data/user1/yy
|
||||
mkdir -p logs
|
||||
|
||||
VENV="${VENV_VLLM}"
|
||||
export PATH="$VENV/bin:$PATH"
|
||||
export PYTHONUNBUFFERED=1
|
||||
export TMPDIR=/data/user1/yy/tmp
|
||||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
|
||||
|
||||
LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||
PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||
|
||||
rm -f "$PID_FILE"
|
||||
|
||||
# Build the base command as an array.
|
||||
SERVER_ARGS=(
|
||||
vllm serve "$MODEL_PATH"
|
||||
--trust-remote-code
|
||||
--tensor-parallel-size "$TP"
|
||||
--kv-cache-dtype fp8
|
||||
--max-model-len "$MAX_MODEL_LEN"
|
||||
--max-num-seqs "$MAX_NUM_SEQS"
|
||||
--block-size 256
|
||||
--gpu-memory-utilization 0.90
|
||||
--tokenizer-mode deepseek_v4
|
||||
--reasoning-parser deepseek_v4
|
||||
--no-disable-hybrid-kv-cache-manager
|
||||
--disable-uvicorn-access-log
|
||||
--port "$VLLM_PORT"
|
||||
)
|
||||
|
||||
if [[ "$DP" -gt 1 ]]; then
|
||||
SERVER_ARGS+=(
|
||||
--data-parallel-size "$DP"
|
||||
--data-parallel-size-local "$DP"
|
||||
--data-parallel-multi-port-external-lb
|
||||
)
|
||||
# vLLM 0.24.0 hard-codes the DP supervisor port to 9256 in
|
||||
# entrypoints/openai/cli_args.py. The bench client and health checks must
|
||||
# target that port when DP > 1.
|
||||
HEALTH_PORT="$VLLM_DP_SUPERVISOR_PORT"
|
||||
else
|
||||
HEALTH_PORT="$VLLM_PORT"
|
||||
fi
|
||||
|
||||
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||
|
||||
echo "=== Starting vLLM server (TP=${TP}, DP=${DP}, max_model_len=${MAX_MODEL_LEN}, max_num_seqs=${MAX_NUM_SEQS}) ==="
|
||||
echo "Model: $MODEL_PATH"
|
||||
echo "Health port: $HEALTH_PORT"
|
||||
echo "Command: $SERVER_ARGS_STR"
|
||||
echo "Log: $LOG"
|
||||
|
||||
nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
|
||||
|
||||
PID=$!
|
||||
echo $PID > "$PID_FILE"
|
||||
echo "PID: $PID"
|
||||
echo "Waiting for health on port ${HEALTH_PORT}..."
|
||||
|
||||
for i in $(seq 1 240); do
|
||||
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${HEALTH_PORT}/health" >/dev/null 2>&1; then
|
||||
echo "vLLM server is ready at http://127.0.0.1:${HEALTH_PORT}"
|
||||
echo "Log: $LOG"
|
||||
exit 0
|
||||
fi
|
||||
if ! kill -0 $PID 2>/dev/null; then
|
||||
echo "ERROR: vLLM server exited early"
|
||||
tail -200 "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
echo "Waiting... ($i/240)"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "ERROR: vLLM server not healthy after 240 retries"
|
||||
tail -200 "$LOG"
|
||||
exit 1
|
||||
@ -93,6 +93,72 @@ def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: floa
|
||||
}
|
||||
|
||||
|
||||
def parse_gpu_memory_csv(jsonl_path: Path) -> dict | None:
|
||||
"""Parse a paired nvidia-smi CSV for GPU memory/utilization statistics.
|
||||
|
||||
The CSV is expected to live in a sibling `gpu_logs/` directory or in the same
|
||||
`raw_outputs/` directory, named `gpu_mem_c<conc>_i<isl>_o<dsl>.csv`.
|
||||
"""
|
||||
result_root = jsonl_path.parent.parent
|
||||
scenario_id = jsonl_path.stem.split("_", 2)[2] # e.g. c1_i1024_o128
|
||||
csv_name = f"gpu_mem_{scenario_id}.csv"
|
||||
|
||||
csv_path = None
|
||||
for candidate in (
|
||||
result_root / "gpu_logs" / csv_name,
|
||||
result_root / "raw_outputs" / csv_name,
|
||||
):
|
||||
if candidate.exists() and candidate.stat().st_size > 0:
|
||||
csv_path = candidate
|
||||
break
|
||||
|
||||
if csv_path is None:
|
||||
return None
|
||||
|
||||
per_gpu = {}
|
||||
total_mb = None
|
||||
try:
|
||||
with open(csv_path, "r", encoding="utf-8") as f:
|
||||
header = f.readline()
|
||||
if not header.strip():
|
||||
return None
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = [p.strip() for p in line.split(",")]
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
idx = parts[1]
|
||||
try:
|
||||
used = float(parts[2].split()[0])
|
||||
total = float(parts[3].split()[0])
|
||||
util = float(parts[4].split()[0])
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
per_gpu.setdefault(idx, {"used": [], "util": []})
|
||||
per_gpu[idx]["used"].append(used)
|
||||
per_gpu[idx]["util"].append(util)
|
||||
if total_mb is None:
|
||||
total_mb = total
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if not per_gpu or total_mb is None:
|
||||
return None
|
||||
|
||||
peak_used = max(max(g["used"]) for g in per_gpu.values())
|
||||
avg_used = sum(sum(g["used"]) / len(g["used"]) for g in per_gpu.values()) / len(per_gpu)
|
||||
peak_util = max(max(g["util"]) for g in per_gpu.values())
|
||||
|
||||
return {
|
||||
"peak_used_mb": peak_used,
|
||||
"avg_used_mb": avg_used,
|
||||
"peak_utilization_pct": peak_util,
|
||||
"memory_total_mb": total_mb,
|
||||
}
|
||||
|
||||
|
||||
def generate_report(result_root: Path, backend: str, scenarios: list[dict], metadata: dict | None) -> None:
|
||||
report_path = result_root / "report.md"
|
||||
model = metadata.get("model", "unknown") if metadata else "unknown"
|
||||
@ -106,13 +172,18 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta
|
||||
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
|
||||
|
||||
f.write("## Results\n\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | Peak GPU mem | SLO |\n")
|
||||
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
|
||||
|
||||
for s in scenarios:
|
||||
cfg = s["config"]
|
||||
m = s["metrics"]
|
||||
slo = s.get("slo_status", {}).get("overall", "")
|
||||
gpu = m.get("gpu_memory")
|
||||
if gpu:
|
||||
gpu_str = f"{gpu['peak_used_mb']:.0f}/{gpu['memory_total_mb']:.0f} MiB ({100*gpu['peak_used_mb']/gpu['memory_total_mb']:.1f}%)"
|
||||
else:
|
||||
gpu_str = "-"
|
||||
f.write(
|
||||
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
|
||||
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
|
||||
@ -120,7 +191,7 @@ def generate_report(result_root: Path, backend: str, scenarios: list[dict], meta
|
||||
f"{m['total_token_throughput']:.2f} | "
|
||||
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
|
||||
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {gpu_str} | {slo} |\n"
|
||||
)
|
||||
f.write("\n")
|
||||
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
|
||||
@ -152,10 +223,13 @@ def main() -> None:
|
||||
raise SystemExit("Could not infer backend from raw outputs")
|
||||
|
||||
metadata = None
|
||||
old_scenarios = []
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
try:
|
||||
metadata = json.load(f).get("metadata")
|
||||
existing = json.load(f)
|
||||
metadata = existing.get("metadata")
|
||||
old_scenarios = existing.get("scenarios", [])
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
@ -183,6 +257,7 @@ def main() -> None:
|
||||
continue
|
||||
|
||||
metrics = compute_metrics(data)
|
||||
metrics["gpu_memory"] = parse_gpu_memory_csv(jsonl_path)
|
||||
scenario = {
|
||||
"name": scenario_name(concurrency, input_len, output_len),
|
||||
"config": {
|
||||
@ -199,10 +274,22 @@ def main() -> None:
|
||||
}
|
||||
scenarios.append(scenario)
|
||||
|
||||
if not scenarios:
|
||||
if not scenarios and not old_scenarios:
|
||||
print("No benchmark outputs found to parse")
|
||||
return
|
||||
|
||||
# Preserve any manually-recorded skipped/failed scenarios (e.g. optional
|
||||
# combinations that OOMed) that do not have a fresh raw output.
|
||||
parsed_names = {s["name"] for s in scenarios}
|
||||
for old in old_scenarios:
|
||||
if old.get("status") and old["name"] not in parsed_names:
|
||||
scenarios.append(old)
|
||||
scenarios.sort(key=lambda s: (
|
||||
s["config"]["input_len"],
|
||||
s["config"]["output_len"],
|
||||
s["config"]["concurrency"],
|
||||
))
|
||||
|
||||
if results_json.exists():
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user