fix: replace 5 buggy parse_results.py (raw_requests bloat) with shared parse_backend.py wrapper
This commit is contained in:
yy-fighting
parent
489c5a5e61
commit
3b0297516e
@ -1,312 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
||||
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||
|
||||
Usage:
|
||||
python3 parse_results.py <result_root>
|
||||
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import OrderedDict
|
||||
|
||||
METRIC_PATTERNS = OrderedDict(
|
||||
[
|
||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_summary_log(log_text: str) -> dict:
|
||||
result = {}
|
||||
for key, patterns in METRIC_PATTERNS.items():
|
||||
result[key] = find_float(log_text, patterns)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
k = (len(sorted_values) - 1) * p / 100.0
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_values) - 1)
|
||||
if f == c:
|
||||
return sorted_values[f]
|
||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
||||
|
||||
|
||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
||||
"""Parse sglang.bench_serving --output-file output.
|
||||
|
||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
||||
JSON object per request. Returns (raw_requests, aggregates).
|
||||
"""
|
||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return [], {}
|
||||
|
||||
# Try single aggregate JSON object first.
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
||||
aggregates = {
|
||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
||||
"failed": 0,
|
||||
"input_tokens": data.get("total_input_tokens", 0),
|
||||
"output_tokens": data.get("total_output_tokens", 0),
|
||||
"latencies": {
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms"),
|
||||
"p50": data.get("median_e2e_latency_ms"),
|
||||
"p90": data.get("p90_e2e_latency_ms"),
|
||||
"p95": None,
|
||||
"p99": data.get("p99_e2e_latency_ms"),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms"),
|
||||
"p50": data.get("median_ttft_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_ttft_ms"),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms"),
|
||||
"p50": data.get("median_tpot_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_tpot_ms"),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms"),
|
||||
"p50": data.get("median_itl_ms"),
|
||||
"p90": None,
|
||||
"p95": data.get("p95_itl_ms"),
|
||||
"p99": data.get("p99_itl_ms"),
|
||||
},
|
||||
},
|
||||
}
|
||||
return [data], aggregates
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fall back to JSONL per-request parsing.
|
||||
raw_requests = []
|
||||
ttfts = []
|
||||
tpots = []
|
||||
itls = []
|
||||
e2es = []
|
||||
input_tokens = []
|
||||
output_tokens = []
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
raw_requests.append(req)
|
||||
|
||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
||||
|
||||
if ttft:
|
||||
ttfts.append(float(ttft))
|
||||
if tpot:
|
||||
tpots.append(float(tpot))
|
||||
if itl:
|
||||
itls.append(float(itl))
|
||||
if e2e:
|
||||
e2es.append(float(e2e))
|
||||
if in_tok:
|
||||
input_tokens.append(int(in_tok))
|
||||
if out_tok:
|
||||
output_tokens.append(int(out_tok))
|
||||
|
||||
if req.get("success", True):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def latency_bucket(values: list[float]) -> dict:
|
||||
if not values:
|
||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
||||
return {
|
||||
"mean": round(sum(values) / len(values), 2),
|
||||
"p50": round(percentile(values, 50), 2),
|
||||
"p90": round(percentile(values, 90), 2),
|
||||
"p95": round(percentile(values, 95), 2),
|
||||
"p99": round(percentile(values, 99), 2),
|
||||
}
|
||||
|
||||
aggregates = {
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"input_tokens": sum(input_tokens),
|
||||
"output_tokens": sum(output_tokens),
|
||||
"latencies": {
|
||||
"e2e_ms": latency_bucket(e2es),
|
||||
"ttft_ms": latency_bucket(ttfts),
|
||||
"tpot_ms": latency_bucket(tpots),
|
||||
"itl_ms": latency_bucket(itls),
|
||||
},
|
||||
}
|
||||
|
||||
return raw_requests, aggregates
|
||||
|
||||
|
||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
||||
"""Parse one raw output file into a scenario dict."""
|
||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
parts = raw_file.stem.split("_")
|
||||
if len(parts) < 6:
|
||||
return None
|
||||
try:
|
||||
concurrency = int(parts[-3])
|
||||
input_len = int(parts[-2])
|
||||
output_len = int(parts[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
||||
summary = {}
|
||||
if log_file.exists():
|
||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
_, aggregates = parse_jsonl(raw_file)
|
||||
|
||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
||||
# raw output file. Override them from the human-readable summary log when
|
||||
# it is available.
|
||||
if summary.get("successful_requests") is not None:
|
||||
aggregates["success"] = int(summary["successful_requests"])
|
||||
# The summary log only reports successes; assume failures are zero
|
||||
# unless the aggregate JSON already provided a non-zero failed count.
|
||||
if not aggregates.get("failed"):
|
||||
aggregates["failed"] = 0
|
||||
|
||||
scenario = {
|
||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"success": aggregates["success"],
|
||||
"failed": aggregates["failed"],
|
||||
"duration_s": summary.get("benchmark_duration_s"),
|
||||
"request_throughput": summary.get("request_throughput"),
|
||||
"input_token_throughput": summary.get("input_token_throughput"),
|
||||
"output_token_throughput": summary.get("output_token_throughput"),
|
||||
"total_token_throughput": summary.get("total_token_throughput"),
|
||||
"accept_length": None,
|
||||
"latencies": aggregates["latencies"],
|
||||
}
|
||||
|
||||
return scenario
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
json_path = result_root / "results.json"
|
||||
report_path = result_root / "report.md"
|
||||
|
||||
if not json_path.exists():
|
||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
||||
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scenarios = []
|
||||
if raw_dir.exists():
|
||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
||||
scenario = parse_scenario(result_root, raw_file)
|
||||
if scenario:
|
||||
scenarios.append(scenario)
|
||||
|
||||
data["scenarios"] = scenarios
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate report.md
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
meta = data["metadata"]
|
||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
||||
f.write("## Metadata\n\n")
|
||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
||||
f.write(f"- **Model**: {meta['model']}\n")
|
||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
||||
f.write("\n## Results\n\n")
|
||||
f.write(
|
||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
||||
)
|
||||
f.write(
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
||||
)
|
||||
for s in scenarios:
|
||||
lat = s["latencies"]
|
||||
f.write(
|
||||
f"| {s['name']} "
|
||||
f"| {s['concurrency']} "
|
||||
f"| {s['input_len']}/{s['output_len']} "
|
||||
f"| {s['success']} "
|
||||
f"| {s['failed']} "
|
||||
f"| {s.get('request_throughput') or ''} "
|
||||
f"| {s.get('output_token_throughput') or ''} "
|
||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
||||
)
|
||||
|
||||
print(f"Updated {json_path}")
|
||||
print(f"Wrote {report_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_d = _HERE
|
||||
while True:
|
||||
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||
if os.path.isfile(cand):
|
||||
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||
parent = os.path.dirname(_d)
|
||||
if parent == _d:
|
||||
break
|
||||
_d = parent
|
||||
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@ -1,312 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
||||
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||
|
||||
Usage:
|
||||
python3 parse_results.py <result_root>
|
||||
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import OrderedDict
|
||||
|
||||
METRIC_PATTERNS = OrderedDict(
|
||||
[
|
||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_summary_log(log_text: str) -> dict:
|
||||
result = {}
|
||||
for key, patterns in METRIC_PATTERNS.items():
|
||||
result[key] = find_float(log_text, patterns)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
k = (len(sorted_values) - 1) * p / 100.0
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_values) - 1)
|
||||
if f == c:
|
||||
return sorted_values[f]
|
||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
||||
|
||||
|
||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
||||
"""Parse sglang.bench_serving --output-file output.
|
||||
|
||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
||||
JSON object per request. Returns (raw_requests, aggregates).
|
||||
"""
|
||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return [], {}
|
||||
|
||||
# Try single aggregate JSON object first.
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
||||
aggregates = {
|
||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
||||
"failed": 0,
|
||||
"input_tokens": data.get("total_input_tokens", 0),
|
||||
"output_tokens": data.get("total_output_tokens", 0),
|
||||
"latencies": {
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms"),
|
||||
"p50": data.get("median_e2e_latency_ms"),
|
||||
"p90": data.get("p90_e2e_latency_ms"),
|
||||
"p95": None,
|
||||
"p99": data.get("p99_e2e_latency_ms"),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms"),
|
||||
"p50": data.get("median_ttft_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_ttft_ms"),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms"),
|
||||
"p50": data.get("median_tpot_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_tpot_ms"),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms"),
|
||||
"p50": data.get("median_itl_ms"),
|
||||
"p90": None,
|
||||
"p95": data.get("p95_itl_ms"),
|
||||
"p99": data.get("p99_itl_ms"),
|
||||
},
|
||||
},
|
||||
}
|
||||
return [data], aggregates
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fall back to JSONL per-request parsing.
|
||||
raw_requests = []
|
||||
ttfts = []
|
||||
tpots = []
|
||||
itls = []
|
||||
e2es = []
|
||||
input_tokens = []
|
||||
output_tokens = []
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
raw_requests.append(req)
|
||||
|
||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
||||
|
||||
if ttft:
|
||||
ttfts.append(float(ttft))
|
||||
if tpot:
|
||||
tpots.append(float(tpot))
|
||||
if itl:
|
||||
itls.append(float(itl))
|
||||
if e2e:
|
||||
e2es.append(float(e2e))
|
||||
if in_tok:
|
||||
input_tokens.append(int(in_tok))
|
||||
if out_tok:
|
||||
output_tokens.append(int(out_tok))
|
||||
|
||||
if req.get("success", True):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def latency_bucket(values: list[float]) -> dict:
|
||||
if not values:
|
||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
||||
return {
|
||||
"mean": round(sum(values) / len(values), 2),
|
||||
"p50": round(percentile(values, 50), 2),
|
||||
"p90": round(percentile(values, 90), 2),
|
||||
"p95": round(percentile(values, 95), 2),
|
||||
"p99": round(percentile(values, 99), 2),
|
||||
}
|
||||
|
||||
aggregates = {
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"input_tokens": sum(input_tokens),
|
||||
"output_tokens": sum(output_tokens),
|
||||
"latencies": {
|
||||
"e2e_ms": latency_bucket(e2es),
|
||||
"ttft_ms": latency_bucket(ttfts),
|
||||
"tpot_ms": latency_bucket(tpots),
|
||||
"itl_ms": latency_bucket(itls),
|
||||
},
|
||||
}
|
||||
|
||||
return raw_requests, aggregates
|
||||
|
||||
|
||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
||||
"""Parse one raw output file into a scenario dict."""
|
||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
parts = raw_file.stem.split("_")
|
||||
if len(parts) < 6:
|
||||
return None
|
||||
try:
|
||||
concurrency = int(parts[-3])
|
||||
input_len = int(parts[-2])
|
||||
output_len = int(parts[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
||||
summary = {}
|
||||
if log_file.exists():
|
||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
_, aggregates = parse_jsonl(raw_file)
|
||||
|
||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
||||
# raw output file. Override them from the human-readable summary log when
|
||||
# it is available.
|
||||
if summary.get("successful_requests") is not None:
|
||||
aggregates["success"] = int(summary["successful_requests"])
|
||||
# The summary log only reports successes; assume failures are zero
|
||||
# unless the aggregate JSON already provided a non-zero failed count.
|
||||
if not aggregates.get("failed"):
|
||||
aggregates["failed"] = 0
|
||||
|
||||
scenario = {
|
||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"success": aggregates["success"],
|
||||
"failed": aggregates["failed"],
|
||||
"duration_s": summary.get("benchmark_duration_s"),
|
||||
"request_throughput": summary.get("request_throughput"),
|
||||
"input_token_throughput": summary.get("input_token_throughput"),
|
||||
"output_token_throughput": summary.get("output_token_throughput"),
|
||||
"total_token_throughput": summary.get("total_token_throughput"),
|
||||
"accept_length": None,
|
||||
"latencies": aggregates["latencies"],
|
||||
}
|
||||
|
||||
return scenario
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
json_path = result_root / "results.json"
|
||||
report_path = result_root / "report.md"
|
||||
|
||||
if not json_path.exists():
|
||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
||||
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scenarios = []
|
||||
if raw_dir.exists():
|
||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
||||
scenario = parse_scenario(result_root, raw_file)
|
||||
if scenario:
|
||||
scenarios.append(scenario)
|
||||
|
||||
data["scenarios"] = scenarios
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate report.md
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
meta = data["metadata"]
|
||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
||||
f.write("## Metadata\n\n")
|
||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
||||
f.write(f"- **Model**: {meta['model']}\n")
|
||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
||||
f.write("\n## Results\n\n")
|
||||
f.write(
|
||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
||||
)
|
||||
f.write(
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
||||
)
|
||||
for s in scenarios:
|
||||
lat = s["latencies"]
|
||||
f.write(
|
||||
f"| {s['name']} "
|
||||
f"| {s['concurrency']} "
|
||||
f"| {s['input_len']}/{s['output_len']} "
|
||||
f"| {s['success']} "
|
||||
f"| {s['failed']} "
|
||||
f"| {s.get('request_throughput') or ''} "
|
||||
f"| {s.get('output_token_throughput') or ''} "
|
||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
||||
)
|
||||
|
||||
print(f"Updated {json_path}")
|
||||
print(f"Wrote {report_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_d = _HERE
|
||||
while True:
|
||||
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||
if os.path.isfile(cand):
|
||||
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||
parent = os.path.dirname(_d)
|
||||
if parent == _d:
|
||||
break
|
||||
_d = parent
|
||||
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@ -1,312 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
||||
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||
|
||||
Usage:
|
||||
python3 parse_results.py <result_root>
|
||||
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import OrderedDict
|
||||
|
||||
METRIC_PATTERNS = OrderedDict(
|
||||
[
|
||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_summary_log(log_text: str) -> dict:
|
||||
result = {}
|
||||
for key, patterns in METRIC_PATTERNS.items():
|
||||
result[key] = find_float(log_text, patterns)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
k = (len(sorted_values) - 1) * p / 100.0
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_values) - 1)
|
||||
if f == c:
|
||||
return sorted_values[f]
|
||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
||||
|
||||
|
||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
||||
"""Parse sglang.bench_serving --output-file output.
|
||||
|
||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
||||
JSON object per request. Returns (raw_requests, aggregates).
|
||||
"""
|
||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return [], {}
|
||||
|
||||
# Try single aggregate JSON object first.
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
||||
aggregates = {
|
||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
||||
"failed": 0,
|
||||
"input_tokens": data.get("total_input_tokens", 0),
|
||||
"output_tokens": data.get("total_output_tokens", 0),
|
||||
"latencies": {
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms"),
|
||||
"p50": data.get("median_e2e_latency_ms"),
|
||||
"p90": data.get("p90_e2e_latency_ms"),
|
||||
"p95": None,
|
||||
"p99": data.get("p99_e2e_latency_ms"),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms"),
|
||||
"p50": data.get("median_ttft_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_ttft_ms"),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms"),
|
||||
"p50": data.get("median_tpot_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_tpot_ms"),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms"),
|
||||
"p50": data.get("median_itl_ms"),
|
||||
"p90": None,
|
||||
"p95": data.get("p95_itl_ms"),
|
||||
"p99": data.get("p99_itl_ms"),
|
||||
},
|
||||
},
|
||||
}
|
||||
return [data], aggregates
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fall back to JSONL per-request parsing.
|
||||
raw_requests = []
|
||||
ttfts = []
|
||||
tpots = []
|
||||
itls = []
|
||||
e2es = []
|
||||
input_tokens = []
|
||||
output_tokens = []
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
raw_requests.append(req)
|
||||
|
||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
||||
|
||||
if ttft:
|
||||
ttfts.append(float(ttft))
|
||||
if tpot:
|
||||
tpots.append(float(tpot))
|
||||
if itl:
|
||||
itls.append(float(itl))
|
||||
if e2e:
|
||||
e2es.append(float(e2e))
|
||||
if in_tok:
|
||||
input_tokens.append(int(in_tok))
|
||||
if out_tok:
|
||||
output_tokens.append(int(out_tok))
|
||||
|
||||
if req.get("success", True):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def latency_bucket(values: list[float]) -> dict:
|
||||
if not values:
|
||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
||||
return {
|
||||
"mean": round(sum(values) / len(values), 2),
|
||||
"p50": round(percentile(values, 50), 2),
|
||||
"p90": round(percentile(values, 90), 2),
|
||||
"p95": round(percentile(values, 95), 2),
|
||||
"p99": round(percentile(values, 99), 2),
|
||||
}
|
||||
|
||||
aggregates = {
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"input_tokens": sum(input_tokens),
|
||||
"output_tokens": sum(output_tokens),
|
||||
"latencies": {
|
||||
"e2e_ms": latency_bucket(e2es),
|
||||
"ttft_ms": latency_bucket(ttfts),
|
||||
"tpot_ms": latency_bucket(tpots),
|
||||
"itl_ms": latency_bucket(itls),
|
||||
},
|
||||
}
|
||||
|
||||
return raw_requests, aggregates
|
||||
|
||||
|
||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
||||
"""Parse one raw output file into a scenario dict."""
|
||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
parts = raw_file.stem.split("_")
|
||||
if len(parts) < 6:
|
||||
return None
|
||||
try:
|
||||
concurrency = int(parts[-3])
|
||||
input_len = int(parts[-2])
|
||||
output_len = int(parts[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
||||
summary = {}
|
||||
if log_file.exists():
|
||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
_, aggregates = parse_jsonl(raw_file)
|
||||
|
||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
||||
# raw output file. Override them from the human-readable summary log when
|
||||
# it is available.
|
||||
if summary.get("successful_requests") is not None:
|
||||
aggregates["success"] = int(summary["successful_requests"])
|
||||
# The summary log only reports successes; assume failures are zero
|
||||
# unless the aggregate JSON already provided a non-zero failed count.
|
||||
if not aggregates.get("failed"):
|
||||
aggregates["failed"] = 0
|
||||
|
||||
scenario = {
|
||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"success": aggregates["success"],
|
||||
"failed": aggregates["failed"],
|
||||
"duration_s": summary.get("benchmark_duration_s"),
|
||||
"request_throughput": summary.get("request_throughput"),
|
||||
"input_token_throughput": summary.get("input_token_throughput"),
|
||||
"output_token_throughput": summary.get("output_token_throughput"),
|
||||
"total_token_throughput": summary.get("total_token_throughput"),
|
||||
"accept_length": None,
|
||||
"latencies": aggregates["latencies"],
|
||||
}
|
||||
|
||||
return scenario
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
json_path = result_root / "results.json"
|
||||
report_path = result_root / "report.md"
|
||||
|
||||
if not json_path.exists():
|
||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
||||
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scenarios = []
|
||||
if raw_dir.exists():
|
||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
||||
scenario = parse_scenario(result_root, raw_file)
|
||||
if scenario:
|
||||
scenarios.append(scenario)
|
||||
|
||||
data["scenarios"] = scenarios
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate report.md
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
meta = data["metadata"]
|
||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
||||
f.write("## Metadata\n\n")
|
||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
||||
f.write(f"- **Model**: {meta['model']}\n")
|
||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
||||
f.write("\n## Results\n\n")
|
||||
f.write(
|
||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
||||
)
|
||||
f.write(
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
||||
)
|
||||
for s in scenarios:
|
||||
lat = s["latencies"]
|
||||
f.write(
|
||||
f"| {s['name']} "
|
||||
f"| {s['concurrency']} "
|
||||
f"| {s['input_len']}/{s['output_len']} "
|
||||
f"| {s['success']} "
|
||||
f"| {s['failed']} "
|
||||
f"| {s.get('request_throughput') or ''} "
|
||||
f"| {s.get('output_token_throughput') or ''} "
|
||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
||||
)
|
||||
|
||||
print(f"Updated {json_path}")
|
||||
print(f"Wrote {report_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_d = _HERE
|
||||
while True:
|
||||
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||
if os.path.isfile(cand):
|
||||
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||
parent = os.path.dirname(_d)
|
||||
if parent == _d:
|
||||
break
|
||||
_d = parent
|
||||
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
22
experiments/p800/qwen3_8b_p800_sglang_tp8/parse_results.py
Executable file
22
experiments/p800/qwen3_8b_p800_sglang_tp8/parse_results.py
Executable file
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||
|
||||
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||
"""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_d = _HERE
|
||||
while True:
|
||||
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||
if os.path.isfile(cand):
|
||||
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||
parent = os.path.dirname(_d)
|
||||
if parent == _d:
|
||||
break
|
||||
_d = parent
|
||||
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@ -1,313 +1,22 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
||||
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||
|
||||
Usage:
|
||||
python3 parse_results.py <result_root>
|
||||
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import OrderedDict
|
||||
|
||||
METRIC_PATTERNS = OrderedDict(
|
||||
[
|
||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
||||
for pat in patterns:
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def parse_summary_log(log_text: str) -> dict:
|
||||
result = {}
|
||||
for key, patterns in METRIC_PATTERNS.items():
|
||||
result[key] = find_float(log_text, patterns)
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
k = (len(sorted_values) - 1) * p / 100.0
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_values) - 1)
|
||||
if f == c:
|
||||
return sorted_values[f]
|
||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
||||
|
||||
|
||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
||||
"""Parse sglang.bench_serving --output-file output.
|
||||
|
||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
||||
JSON object per request. Returns (raw_requests, aggregates).
|
||||
"""
|
||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
||||
if not text:
|
||||
return [], {}
|
||||
|
||||
# Try single aggregate JSON object first.
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
||||
aggregates = {
|
||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
||||
"failed": 0,
|
||||
"input_tokens": data.get("total_input_tokens", 0),
|
||||
"output_tokens": data.get("total_output_tokens", 0),
|
||||
"latencies": {
|
||||
"e2e_ms": {
|
||||
"mean": data.get("mean_e2e_latency_ms"),
|
||||
"p50": data.get("median_e2e_latency_ms"),
|
||||
"p90": data.get("p90_e2e_latency_ms"),
|
||||
"p95": None,
|
||||
"p99": data.get("p99_e2e_latency_ms"),
|
||||
},
|
||||
"ttft_ms": {
|
||||
"mean": data.get("mean_ttft_ms"),
|
||||
"p50": data.get("median_ttft_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_ttft_ms"),
|
||||
},
|
||||
"tpot_ms": {
|
||||
"mean": data.get("mean_tpot_ms"),
|
||||
"p50": data.get("median_tpot_ms"),
|
||||
"p90": None,
|
||||
"p95": None,
|
||||
"p99": data.get("p99_tpot_ms"),
|
||||
},
|
||||
"itl_ms": {
|
||||
"mean": data.get("mean_itl_ms"),
|
||||
"p50": data.get("median_itl_ms"),
|
||||
"p90": None,
|
||||
"p95": data.get("p95_itl_ms"),
|
||||
"p99": data.get("p99_itl_ms"),
|
||||
},
|
||||
},
|
||||
}
|
||||
return [data], aggregates
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Fall back to JSONL per-request parsing.
|
||||
raw_requests = []
|
||||
ttfts = []
|
||||
tpots = []
|
||||
itls = []
|
||||
e2es = []
|
||||
input_tokens = []
|
||||
output_tokens = []
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
raw_requests.append(req)
|
||||
|
||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
||||
|
||||
if ttft:
|
||||
ttfts.append(float(ttft))
|
||||
if tpot:
|
||||
tpots.append(float(tpot))
|
||||
if itl:
|
||||
itls.append(float(itl))
|
||||
if e2e:
|
||||
e2es.append(float(e2e))
|
||||
if in_tok:
|
||||
input_tokens.append(int(in_tok))
|
||||
if out_tok:
|
||||
output_tokens.append(int(out_tok))
|
||||
|
||||
if req.get("success", True):
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def latency_bucket(values: list[float]) -> dict:
|
||||
if not values:
|
||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
||||
return {
|
||||
"mean": round(sum(values) / len(values), 2),
|
||||
"p50": round(percentile(values, 50), 2),
|
||||
"p90": round(percentile(values, 90), 2),
|
||||
"p95": round(percentile(values, 95), 2),
|
||||
"p99": round(percentile(values, 99), 2),
|
||||
}
|
||||
|
||||
aggregates = {
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"input_tokens": sum(input_tokens),
|
||||
"output_tokens": sum(output_tokens),
|
||||
"latencies": {
|
||||
"e2e_ms": latency_bucket(e2es),
|
||||
"ttft_ms": latency_bucket(ttfts),
|
||||
"tpot_ms": latency_bucket(tpots),
|
||||
"itl_ms": latency_bucket(itls),
|
||||
},
|
||||
}
|
||||
|
||||
return raw_requests, aggregates
|
||||
|
||||
|
||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
||||
"""Parse one raw output file into a scenario dict."""
|
||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
||||
parts = raw_file.stem.split("_")
|
||||
if len(parts) < 6:
|
||||
return None
|
||||
try:
|
||||
concurrency = int(parts[-3])
|
||||
input_len = int(parts[-2])
|
||||
output_len = int(parts[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
||||
summary = {}
|
||||
if log_file.exists():
|
||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
||||
|
||||
raw_requests, aggregates = parse_jsonl(raw_file)
|
||||
|
||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
||||
# raw output file. Override them from the human-readable summary log when
|
||||
# it is available.
|
||||
if summary.get("successful_requests") is not None:
|
||||
aggregates["success"] = int(summary["successful_requests"])
|
||||
# The summary log only reports successes; assume failures are zero
|
||||
# unless the aggregate JSON already provided a non-zero failed count.
|
||||
if not aggregates.get("failed"):
|
||||
aggregates["failed"] = 0
|
||||
|
||||
scenario = {
|
||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
||||
"concurrency": concurrency,
|
||||
"input_len": input_len,
|
||||
"output_len": output_len,
|
||||
"success": aggregates["success"],
|
||||
"failed": aggregates["failed"],
|
||||
"duration_s": summary.get("benchmark_duration_s"),
|
||||
"request_throughput": summary.get("request_throughput"),
|
||||
"input_token_throughput": summary.get("input_token_throughput"),
|
||||
"output_token_throughput": summary.get("output_token_throughput"),
|
||||
"total_token_throughput": summary.get("total_token_throughput"),
|
||||
"accept_length": None,
|
||||
"latencies": aggregates["latencies"],
|
||||
"raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None,
|
||||
}
|
||||
|
||||
return scenario
|
||||
|
||||
|
||||
def main() -> None:
|
||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
||||
raw_dir = result_root / "raw_outputs"
|
||||
json_path = result_root / "results.json"
|
||||
report_path = result_root / "report.md"
|
||||
|
||||
if not json_path.exists():
|
||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
||||
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scenarios = []
|
||||
if raw_dir.exists():
|
||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
||||
scenario = parse_scenario(result_root, raw_file)
|
||||
if scenario:
|
||||
scenarios.append(scenario)
|
||||
|
||||
data["scenarios"] = scenarios
|
||||
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Generate report.md
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
meta = data["metadata"]
|
||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
||||
f.write("## Metadata\n\n")
|
||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
||||
f.write(f"- **Model**: {meta['model']}\n")
|
||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
||||
f.write("\n## Results\n\n")
|
||||
f.write(
|
||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
||||
)
|
||||
f.write(
|
||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
||||
)
|
||||
for s in scenarios:
|
||||
lat = s["latencies"]
|
||||
f.write(
|
||||
f"| {s['name']} "
|
||||
f"| {s['concurrency']} "
|
||||
f"| {s['input_len']}/{s['output_len']} "
|
||||
f"| {s['success']} "
|
||||
f"| {s['failed']} "
|
||||
f"| {s.get('request_throughput') or ''} "
|
||||
f"| {s.get('output_token_throughput') or ''} "
|
||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
||||
)
|
||||
|
||||
print(f"Updated {json_path}")
|
||||
print(f"Wrote {report_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_d = _HERE
|
||||
while True:
|
||||
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||
if os.path.isfile(cand):
|
||||
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||
parent = os.path.dirname(_d)
|
||||
if parent == _d:
|
||||
break
|
||||
_d = parent
|
||||
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Loading…
x
Reference in New Issue
Block a user