Compare commits
4 Commits
6286733f3c
...
3b0297516e
| Author | SHA1 | Date | |
|---|---|---|---|
|
3b0297516e | ||
|
|
489c5a5e61 | ||
|
|
b6dab8e139 | ||
|
|
3ba4968971 |
9
.gitignore
vendored
9
.gitignore
vendored
@ -78,3 +78,12 @@ bench-output/
|
|||||||
# 逐请求原始日志（体积大；汇总见 results.json / report.md）
|
# 逐请求原始日志（体积大；汇总见 results.json / report.md）
|
||||||
experiments/**/raw_outputs/
|
experiments/**/raw_outputs/
|
||||||
**/dummy_sharegpt.json
|
**/dummy_sharegpt.json
|
||||||
|
|
||||||
|
# Virtual environments (keep only docs under envs/)
|
||||||
|
envs/*
|
||||||
|
!envs/README.md
|
||||||
|
!envs/SM120_DSV4_DEPLOYMENT_GUIDE.md
|
||||||
|
!envs/UV_ENV_SETUP.md
|
||||||
|
|
||||||
|
# Tooling artifacts
|
||||||
|
skills-lock.json
|
||||||
|
|||||||
@ -1,312 +1,22 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||||
|
|
||||||
Usage:
|
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||||
python3 parse_results.py <result_root>
|
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
METRIC_PATTERNS = OrderedDict(
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
[
|
_d = _HERE
|
||||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
while True:
|
||||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
if os.path.isfile(cand):
|
||||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
parent = os.path.dirname(_d)
|
||||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
if parent == _d:
|
||||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
break
|
||||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
_d = parent
|
||||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
sys.exit(1)
|
||||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
|
||||||
for pat in patterns:
|
|
||||||
m = re.search(pat, text)
|
|
||||||
if m:
|
|
||||||
try:
|
|
||||||
return float(m.group(1))
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_summary_log(log_text: str) -> dict:
|
|
||||||
result = {}
|
|
||||||
for key, patterns in METRIC_PATTERNS.items():
|
|
||||||
result[key] = find_float(log_text, patterns)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
sorted_values = sorted(values)
|
|
||||||
k = (len(sorted_values) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(sorted_values) - 1)
|
|
||||||
if f == c:
|
|
||||||
return sorted_values[f]
|
|
||||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
|
||||||
"""Parse sglang.bench_serving --output-file output.
|
|
||||||
|
|
||||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
|
||||||
JSON object per request. Returns (raw_requests, aggregates).
|
|
||||||
"""
|
|
||||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
|
||||||
if not text:
|
|
||||||
return [], {}
|
|
||||||
|
|
||||||
# Try single aggregate JSON object first.
|
|
||||||
try:
|
|
||||||
data = json.loads(text)
|
|
||||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
|
||||||
aggregates = {
|
|
||||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
|
||||||
"failed": 0,
|
|
||||||
"input_tokens": data.get("total_input_tokens", 0),
|
|
||||||
"output_tokens": data.get("total_output_tokens", 0),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": data.get("mean_e2e_latency_ms"),
|
|
||||||
"p50": data.get("median_e2e_latency_ms"),
|
|
||||||
"p90": data.get("p90_e2e_latency_ms"),
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_e2e_latency_ms"),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": data.get("mean_ttft_ms"),
|
|
||||||
"p50": data.get("median_ttft_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_ttft_ms"),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": data.get("mean_tpot_ms"),
|
|
||||||
"p50": data.get("median_tpot_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_tpot_ms"),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": data.get("mean_itl_ms"),
|
|
||||||
"p50": data.get("median_itl_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": data.get("p95_itl_ms"),
|
|
||||||
"p99": data.get("p99_itl_ms"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return [data], aggregates
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Fall back to JSONL per-request parsing.
|
|
||||||
raw_requests = []
|
|
||||||
ttfts = []
|
|
||||||
tpots = []
|
|
||||||
itls = []
|
|
||||||
e2es = []
|
|
||||||
input_tokens = []
|
|
||||||
output_tokens = []
|
|
||||||
success = 0
|
|
||||||
failed = 0
|
|
||||||
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
req = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_requests.append(req)
|
|
||||||
|
|
||||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
|
||||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
|
||||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
|
||||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
|
||||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
|
||||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
|
||||||
|
|
||||||
if ttft:
|
|
||||||
ttfts.append(float(ttft))
|
|
||||||
if tpot:
|
|
||||||
tpots.append(float(tpot))
|
|
||||||
if itl:
|
|
||||||
itls.append(float(itl))
|
|
||||||
if e2e:
|
|
||||||
e2es.append(float(e2e))
|
|
||||||
if in_tok:
|
|
||||||
input_tokens.append(int(in_tok))
|
|
||||||
if out_tok:
|
|
||||||
output_tokens.append(int(out_tok))
|
|
||||||
|
|
||||||
if req.get("success", True):
|
|
||||||
success += 1
|
|
||||||
else:
|
|
||||||
failed += 1
|
|
||||||
|
|
||||||
def latency_bucket(values: list[float]) -> dict:
|
|
||||||
if not values:
|
|
||||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
|
||||||
return {
|
|
||||||
"mean": round(sum(values) / len(values), 2),
|
|
||||||
"p50": round(percentile(values, 50), 2),
|
|
||||||
"p90": round(percentile(values, 90), 2),
|
|
||||||
"p95": round(percentile(values, 95), 2),
|
|
||||||
"p99": round(percentile(values, 99), 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
aggregates = {
|
|
||||||
"success": success,
|
|
||||||
"failed": failed,
|
|
||||||
"input_tokens": sum(input_tokens),
|
|
||||||
"output_tokens": sum(output_tokens),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": latency_bucket(e2es),
|
|
||||||
"ttft_ms": latency_bucket(ttfts),
|
|
||||||
"tpot_ms": latency_bucket(tpots),
|
|
||||||
"itl_ms": latency_bucket(itls),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return raw_requests, aggregates
|
|
||||||
|
|
||||||
|
|
||||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
|
||||||
"""Parse one raw output file into a scenario dict."""
|
|
||||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
|
||||||
parts = raw_file.stem.split("_")
|
|
||||||
if len(parts) < 6:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
concurrency = int(parts[-3])
|
|
||||||
input_len = int(parts[-2])
|
|
||||||
output_len = int(parts[-1])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
|
||||||
summary = {}
|
|
||||||
if log_file.exists():
|
|
||||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
|
||||||
|
|
||||||
_, aggregates = parse_jsonl(raw_file)
|
|
||||||
|
|
||||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
|
||||||
# raw output file. Override them from the human-readable summary log when
|
|
||||||
# it is available.
|
|
||||||
if summary.get("successful_requests") is not None:
|
|
||||||
aggregates["success"] = int(summary["successful_requests"])
|
|
||||||
# The summary log only reports successes; assume failures are zero
|
|
||||||
# unless the aggregate JSON already provided a non-zero failed count.
|
|
||||||
if not aggregates.get("failed"):
|
|
||||||
aggregates["failed"] = 0
|
|
||||||
|
|
||||||
scenario = {
|
|
||||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"success": aggregates["success"],
|
|
||||||
"failed": aggregates["failed"],
|
|
||||||
"duration_s": summary.get("benchmark_duration_s"),
|
|
||||||
"request_throughput": summary.get("request_throughput"),
|
|
||||||
"input_token_throughput": summary.get("input_token_throughput"),
|
|
||||||
"output_token_throughput": summary.get("output_token_throughput"),
|
|
||||||
"total_token_throughput": summary.get("total_token_throughput"),
|
|
||||||
"accept_length": None,
|
|
||||||
"latencies": aggregates["latencies"],
|
|
||||||
}
|
|
||||||
|
|
||||||
return scenario
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
json_path = result_root / "results.json"
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
|
|
||||||
if not json_path.exists():
|
|
||||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
|
||||||
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
if raw_dir.exists():
|
|
||||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
|
||||||
scenario = parse_scenario(result_root, raw_file)
|
|
||||||
if scenario:
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
data["scenarios"] = scenarios
|
|
||||||
|
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Generate report.md
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
meta = data["metadata"]
|
|
||||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
|
||||||
f.write("## Metadata\n\n")
|
|
||||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
|
||||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
|
||||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
|
||||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
|
||||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
|
||||||
f.write(f"- **Model**: {meta['model']}\n")
|
|
||||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
|
||||||
f.write("\n## Results\n\n")
|
|
||||||
f.write(
|
|
||||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
|
||||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
|
||||||
)
|
|
||||||
f.write(
|
|
||||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
|
||||||
)
|
|
||||||
for s in scenarios:
|
|
||||||
lat = s["latencies"]
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} "
|
|
||||||
f"| {s['concurrency']} "
|
|
||||||
f"| {s['input_len']}/{s['output_len']} "
|
|
||||||
f"| {s['success']} "
|
|
||||||
f"| {s['failed']} "
|
|
||||||
f"| {s.get('request_throughput') or ''} "
|
|
||||||
f"| {s.get('output_token_throughput') or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Updated {json_path}")
|
|
||||||
print(f"Wrote {report_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,312 +1,22 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||||
|
|
||||||
Usage:
|
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||||
python3 parse_results.py <result_root>
|
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
METRIC_PATTERNS = OrderedDict(
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
[
|
_d = _HERE
|
||||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
while True:
|
||||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
if os.path.isfile(cand):
|
||||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
parent = os.path.dirname(_d)
|
||||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
if parent == _d:
|
||||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
break
|
||||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
_d = parent
|
||||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
sys.exit(1)
|
||||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
|
||||||
for pat in patterns:
|
|
||||||
m = re.search(pat, text)
|
|
||||||
if m:
|
|
||||||
try:
|
|
||||||
return float(m.group(1))
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_summary_log(log_text: str) -> dict:
|
|
||||||
result = {}
|
|
||||||
for key, patterns in METRIC_PATTERNS.items():
|
|
||||||
result[key] = find_float(log_text, patterns)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
sorted_values = sorted(values)
|
|
||||||
k = (len(sorted_values) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(sorted_values) - 1)
|
|
||||||
if f == c:
|
|
||||||
return sorted_values[f]
|
|
||||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
|
||||||
"""Parse sglang.bench_serving --output-file output.
|
|
||||||
|
|
||||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
|
||||||
JSON object per request. Returns (raw_requests, aggregates).
|
|
||||||
"""
|
|
||||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
|
||||||
if not text:
|
|
||||||
return [], {}
|
|
||||||
|
|
||||||
# Try single aggregate JSON object first.
|
|
||||||
try:
|
|
||||||
data = json.loads(text)
|
|
||||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
|
||||||
aggregates = {
|
|
||||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
|
||||||
"failed": 0,
|
|
||||||
"input_tokens": data.get("total_input_tokens", 0),
|
|
||||||
"output_tokens": data.get("total_output_tokens", 0),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": data.get("mean_e2e_latency_ms"),
|
|
||||||
"p50": data.get("median_e2e_latency_ms"),
|
|
||||||
"p90": data.get("p90_e2e_latency_ms"),
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_e2e_latency_ms"),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": data.get("mean_ttft_ms"),
|
|
||||||
"p50": data.get("median_ttft_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_ttft_ms"),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": data.get("mean_tpot_ms"),
|
|
||||||
"p50": data.get("median_tpot_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_tpot_ms"),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": data.get("mean_itl_ms"),
|
|
||||||
"p50": data.get("median_itl_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": data.get("p95_itl_ms"),
|
|
||||||
"p99": data.get("p99_itl_ms"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return [data], aggregates
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Fall back to JSONL per-request parsing.
|
|
||||||
raw_requests = []
|
|
||||||
ttfts = []
|
|
||||||
tpots = []
|
|
||||||
itls = []
|
|
||||||
e2es = []
|
|
||||||
input_tokens = []
|
|
||||||
output_tokens = []
|
|
||||||
success = 0
|
|
||||||
failed = 0
|
|
||||||
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
req = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_requests.append(req)
|
|
||||||
|
|
||||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
|
||||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
|
||||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
|
||||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
|
||||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
|
||||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
|
||||||
|
|
||||||
if ttft:
|
|
||||||
ttfts.append(float(ttft))
|
|
||||||
if tpot:
|
|
||||||
tpots.append(float(tpot))
|
|
||||||
if itl:
|
|
||||||
itls.append(float(itl))
|
|
||||||
if e2e:
|
|
||||||
e2es.append(float(e2e))
|
|
||||||
if in_tok:
|
|
||||||
input_tokens.append(int(in_tok))
|
|
||||||
if out_tok:
|
|
||||||
output_tokens.append(int(out_tok))
|
|
||||||
|
|
||||||
if req.get("success", True):
|
|
||||||
success += 1
|
|
||||||
else:
|
|
||||||
failed += 1
|
|
||||||
|
|
||||||
def latency_bucket(values: list[float]) -> dict:
|
|
||||||
if not values:
|
|
||||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
|
||||||
return {
|
|
||||||
"mean": round(sum(values) / len(values), 2),
|
|
||||||
"p50": round(percentile(values, 50), 2),
|
|
||||||
"p90": round(percentile(values, 90), 2),
|
|
||||||
"p95": round(percentile(values, 95), 2),
|
|
||||||
"p99": round(percentile(values, 99), 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
aggregates = {
|
|
||||||
"success": success,
|
|
||||||
"failed": failed,
|
|
||||||
"input_tokens": sum(input_tokens),
|
|
||||||
"output_tokens": sum(output_tokens),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": latency_bucket(e2es),
|
|
||||||
"ttft_ms": latency_bucket(ttfts),
|
|
||||||
"tpot_ms": latency_bucket(tpots),
|
|
||||||
"itl_ms": latency_bucket(itls),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return raw_requests, aggregates
|
|
||||||
|
|
||||||
|
|
||||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
|
||||||
"""Parse one raw output file into a scenario dict."""
|
|
||||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
|
||||||
parts = raw_file.stem.split("_")
|
|
||||||
if len(parts) < 6:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
concurrency = int(parts[-3])
|
|
||||||
input_len = int(parts[-2])
|
|
||||||
output_len = int(parts[-1])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
|
||||||
summary = {}
|
|
||||||
if log_file.exists():
|
|
||||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
|
||||||
|
|
||||||
_, aggregates = parse_jsonl(raw_file)
|
|
||||||
|
|
||||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
|
||||||
# raw output file. Override them from the human-readable summary log when
|
|
||||||
# it is available.
|
|
||||||
if summary.get("successful_requests") is not None:
|
|
||||||
aggregates["success"] = int(summary["successful_requests"])
|
|
||||||
# The summary log only reports successes; assume failures are zero
|
|
||||||
# unless the aggregate JSON already provided a non-zero failed count.
|
|
||||||
if not aggregates.get("failed"):
|
|
||||||
aggregates["failed"] = 0
|
|
||||||
|
|
||||||
scenario = {
|
|
||||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"success": aggregates["success"],
|
|
||||||
"failed": aggregates["failed"],
|
|
||||||
"duration_s": summary.get("benchmark_duration_s"),
|
|
||||||
"request_throughput": summary.get("request_throughput"),
|
|
||||||
"input_token_throughput": summary.get("input_token_throughput"),
|
|
||||||
"output_token_throughput": summary.get("output_token_throughput"),
|
|
||||||
"total_token_throughput": summary.get("total_token_throughput"),
|
|
||||||
"accept_length": None,
|
|
||||||
"latencies": aggregates["latencies"],
|
|
||||||
}
|
|
||||||
|
|
||||||
return scenario
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
json_path = result_root / "results.json"
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
|
|
||||||
if not json_path.exists():
|
|
||||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
|
||||||
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
if raw_dir.exists():
|
|
||||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
|
||||||
scenario = parse_scenario(result_root, raw_file)
|
|
||||||
if scenario:
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
data["scenarios"] = scenarios
|
|
||||||
|
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Generate report.md
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
meta = data["metadata"]
|
|
||||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
|
||||||
f.write("## Metadata\n\n")
|
|
||||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
|
||||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
|
||||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
|
||||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
|
||||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
|
||||||
f.write(f"- **Model**: {meta['model']}\n")
|
|
||||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
|
||||||
f.write("\n## Results\n\n")
|
|
||||||
f.write(
|
|
||||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
|
||||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
|
||||||
)
|
|
||||||
f.write(
|
|
||||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
|
||||||
)
|
|
||||||
for s in scenarios:
|
|
||||||
lat = s["latencies"]
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} "
|
|
||||||
f"| {s['concurrency']} "
|
|
||||||
f"| {s['input_len']}/{s['output_len']} "
|
|
||||||
f"| {s['success']} "
|
|
||||||
f"| {s['failed']} "
|
|
||||||
f"| {s.get('request_throughput') or ''} "
|
|
||||||
f"| {s.get('output_token_throughput') or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Updated {json_path}")
|
|
||||||
print(f"Wrote {report_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@ -1,312 +1,22 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||||
|
|
||||||
Usage:
|
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||||
python3 parse_results.py <result_root>
|
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
METRIC_PATTERNS = OrderedDict(
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
[
|
_d = _HERE
|
||||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
while True:
|
||||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
if os.path.isfile(cand):
|
||||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
parent = os.path.dirname(_d)
|
||||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
if parent == _d:
|
||||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
break
|
||||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
_d = parent
|
||||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
sys.exit(1)
|
||||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
|
||||||
for pat in patterns:
|
|
||||||
m = re.search(pat, text)
|
|
||||||
if m:
|
|
||||||
try:
|
|
||||||
return float(m.group(1))
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_summary_log(log_text: str) -> dict:
|
|
||||||
result = {}
|
|
||||||
for key, patterns in METRIC_PATTERNS.items():
|
|
||||||
result[key] = find_float(log_text, patterns)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
sorted_values = sorted(values)
|
|
||||||
k = (len(sorted_values) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(sorted_values) - 1)
|
|
||||||
if f == c:
|
|
||||||
return sorted_values[f]
|
|
||||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
|
||||||
"""Parse sglang.bench_serving --output-file output.
|
|
||||||
|
|
||||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
|
||||||
JSON object per request. Returns (raw_requests, aggregates).
|
|
||||||
"""
|
|
||||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
|
||||||
if not text:
|
|
||||||
return [], {}
|
|
||||||
|
|
||||||
# Try single aggregate JSON object first.
|
|
||||||
try:
|
|
||||||
data = json.loads(text)
|
|
||||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
|
||||||
aggregates = {
|
|
||||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
|
||||||
"failed": 0,
|
|
||||||
"input_tokens": data.get("total_input_tokens", 0),
|
|
||||||
"output_tokens": data.get("total_output_tokens", 0),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": data.get("mean_e2e_latency_ms"),
|
|
||||||
"p50": data.get("median_e2e_latency_ms"),
|
|
||||||
"p90": data.get("p90_e2e_latency_ms"),
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_e2e_latency_ms"),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": data.get("mean_ttft_ms"),
|
|
||||||
"p50": data.get("median_ttft_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_ttft_ms"),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": data.get("mean_tpot_ms"),
|
|
||||||
"p50": data.get("median_tpot_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_tpot_ms"),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": data.get("mean_itl_ms"),
|
|
||||||
"p50": data.get("median_itl_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": data.get("p95_itl_ms"),
|
|
||||||
"p99": data.get("p99_itl_ms"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return [data], aggregates
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Fall back to JSONL per-request parsing.
|
|
||||||
raw_requests = []
|
|
||||||
ttfts = []
|
|
||||||
tpots = []
|
|
||||||
itls = []
|
|
||||||
e2es = []
|
|
||||||
input_tokens = []
|
|
||||||
output_tokens = []
|
|
||||||
success = 0
|
|
||||||
failed = 0
|
|
||||||
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
req = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_requests.append(req)
|
|
||||||
|
|
||||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
|
||||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
|
||||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
|
||||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
|
||||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
|
||||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
|
||||||
|
|
||||||
if ttft:
|
|
||||||
ttfts.append(float(ttft))
|
|
||||||
if tpot:
|
|
||||||
tpots.append(float(tpot))
|
|
||||||
if itl:
|
|
||||||
itls.append(float(itl))
|
|
||||||
if e2e:
|
|
||||||
e2es.append(float(e2e))
|
|
||||||
if in_tok:
|
|
||||||
input_tokens.append(int(in_tok))
|
|
||||||
if out_tok:
|
|
||||||
output_tokens.append(int(out_tok))
|
|
||||||
|
|
||||||
if req.get("success", True):
|
|
||||||
success += 1
|
|
||||||
else:
|
|
||||||
failed += 1
|
|
||||||
|
|
||||||
def latency_bucket(values: list[float]) -> dict:
|
|
||||||
if not values:
|
|
||||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
|
||||||
return {
|
|
||||||
"mean": round(sum(values) / len(values), 2),
|
|
||||||
"p50": round(percentile(values, 50), 2),
|
|
||||||
"p90": round(percentile(values, 90), 2),
|
|
||||||
"p95": round(percentile(values, 95), 2),
|
|
||||||
"p99": round(percentile(values, 99), 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
aggregates = {
|
|
||||||
"success": success,
|
|
||||||
"failed": failed,
|
|
||||||
"input_tokens": sum(input_tokens),
|
|
||||||
"output_tokens": sum(output_tokens),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": latency_bucket(e2es),
|
|
||||||
"ttft_ms": latency_bucket(ttfts),
|
|
||||||
"tpot_ms": latency_bucket(tpots),
|
|
||||||
"itl_ms": latency_bucket(itls),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return raw_requests, aggregates
|
|
||||||
|
|
||||||
|
|
||||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
|
||||||
"""Parse one raw output file into a scenario dict."""
|
|
||||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
|
||||||
parts = raw_file.stem.split("_")
|
|
||||||
if len(parts) < 6:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
concurrency = int(parts[-3])
|
|
||||||
input_len = int(parts[-2])
|
|
||||||
output_len = int(parts[-1])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
|
||||||
summary = {}
|
|
||||||
if log_file.exists():
|
|
||||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
|
||||||
|
|
||||||
_, aggregates = parse_jsonl(raw_file)
|
|
||||||
|
|
||||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
|
||||||
# raw output file. Override them from the human-readable summary log when
|
|
||||||
# it is available.
|
|
||||||
if summary.get("successful_requests") is not None:
|
|
||||||
aggregates["success"] = int(summary["successful_requests"])
|
|
||||||
# The summary log only reports successes; assume failures are zero
|
|
||||||
# unless the aggregate JSON already provided a non-zero failed count.
|
|
||||||
if not aggregates.get("failed"):
|
|
||||||
aggregates["failed"] = 0
|
|
||||||
|
|
||||||
scenario = {
|
|
||||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"success": aggregates["success"],
|
|
||||||
"failed": aggregates["failed"],
|
|
||||||
"duration_s": summary.get("benchmark_duration_s"),
|
|
||||||
"request_throughput": summary.get("request_throughput"),
|
|
||||||
"input_token_throughput": summary.get("input_token_throughput"),
|
|
||||||
"output_token_throughput": summary.get("output_token_throughput"),
|
|
||||||
"total_token_throughput": summary.get("total_token_throughput"),
|
|
||||||
"accept_length": None,
|
|
||||||
"latencies": aggregates["latencies"],
|
|
||||||
}
|
|
||||||
|
|
||||||
return scenario
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
json_path = result_root / "results.json"
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
|
|
||||||
if not json_path.exists():
|
|
||||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
|
||||||
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
if raw_dir.exists():
|
|
||||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
|
||||||
scenario = parse_scenario(result_root, raw_file)
|
|
||||||
if scenario:
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
data["scenarios"] = scenarios
|
|
||||||
|
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Generate report.md
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
meta = data["metadata"]
|
|
||||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
|
||||||
f.write("## Metadata\n\n")
|
|
||||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
|
||||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
|
||||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
|
||||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
|
||||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
|
||||||
f.write(f"- **Model**: {meta['model']}\n")
|
|
||||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
|
||||||
f.write("\n## Results\n\n")
|
|
||||||
f.write(
|
|
||||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
|
||||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
|
||||||
)
|
|
||||||
f.write(
|
|
||||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
|
||||||
)
|
|
||||||
for s in scenarios:
|
|
||||||
lat = s["latencies"]
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} "
|
|
||||||
f"| {s['concurrency']} "
|
|
||||||
f"| {s['input_len']}/{s['output_len']} "
|
|
||||||
f"| {s['success']} "
|
|
||||||
f"| {s['failed']} "
|
|
||||||
f"| {s.get('request_throughput') or ''} "
|
|
||||||
f"| {s.get('output_token_throughput') or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Updated {json_path}")
|
|
||||||
print(f"Wrote {report_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
22
experiments/p800/qwen3_8b_p800_sglang_tp8/parse_results.py
Executable file
22
experiments/p800/qwen3_8b_p800_sglang_tp8/parse_results.py
Executable file
@ -0,0 +1,22 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||||
|
|
||||||
|
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||||
|
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
_d = _HERE
|
||||||
|
while True:
|
||||||
|
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||||
|
if os.path.isfile(cand):
|
||||||
|
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||||
|
parent = os.path.dirname(_d)
|
||||||
|
if parent == _d:
|
||||||
|
break
|
||||||
|
_d = parent
|
||||||
|
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
@ -1,313 +1,22 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Parse P800 SGLang benchmark outputs and produce results.json + report.md.
|
"""Thin wrapper: delegate to the shared parser at scripts/common/parse_backend.py.
|
||||||
|
|
||||||
Usage:
|
Replaces per-experiment parse_results.py copies that embedded raw_requests
|
||||||
python3 parse_results.py <result_root>
|
(the results.json bloat bug). Forwards all CLI args to the shared parser.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import re
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
|
||||||
from collections import OrderedDict
|
|
||||||
|
|
||||||
METRIC_PATTERNS = OrderedDict(
|
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
[
|
_d = _HERE
|
||||||
("successful_requests", [r"Successful requests:\s+(\d+)"]),
|
while True:
|
||||||
("benchmark_duration_s", [r"Benchmark duration \(s\):\s+([\d.]+)"]),
|
cand = os.path.join(_d, "scripts", "common", "parse_backend.py")
|
||||||
("request_throughput", [r"Request throughput \(req/s\):\s+([\d.]+)"]),
|
if os.path.isfile(cand):
|
||||||
("input_token_throughput", [r"Input token throughput \(tok/s\):\s+([\d.]+)"]),
|
sys.exit(subprocess.call([sys.executable, cand, *sys.argv[1:]]))
|
||||||
("output_token_throughput", [r"Output token throughput \(tok/s\):\s+([\d.]+)"]),
|
parent = os.path.dirname(_d)
|
||||||
("total_token_throughput", [r"Total token throughput \(tok/s\):\s+([\d.]+)"]),
|
if parent == _d:
|
||||||
("ttft_mean", [r"Mean TTFT \(ms\):\s+([\d.]+)"]),
|
break
|
||||||
("ttft_p50", [r"Median TTFT \(ms\):\s+([\d.]+)", r"P50 TTFT \(ms\):\s+([\d.]+)"]),
|
_d = parent
|
||||||
("ttft_p90", [r"P90 TTFT \(ms\):\s+([\d.]+)"]),
|
print("ERROR: scripts/common/parse_backend.py not found upwards from " + _HERE, file=sys.stderr)
|
||||||
("ttft_p99", [r"P99 TTFT \(ms\):\s+([\d.]+)"]),
|
sys.exit(1)
|
||||||
("tpot_mean", [r"Mean TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p50", [r"Median TPOT \(ms\):\s+([\d.]+)", r"P50 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p90", [r"P90 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("tpot_p99", [r"P99 TPOT \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_mean", [r"Mean E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p50", [r"Median E2E \(ms\):\s+([\d.]+)", r"P50 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p90", [r"P90 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
("e2e_p99", [r"P99 E2E \(ms\):\s+([\d.]+)"]),
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_float(text: str, patterns: list[str]) -> float | None:
|
|
||||||
for pat in patterns:
|
|
||||||
m = re.search(pat, text)
|
|
||||||
if m:
|
|
||||||
try:
|
|
||||||
return float(m.group(1))
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_summary_log(log_text: str) -> dict:
|
|
||||||
result = {}
|
|
||||||
for key, patterns in METRIC_PATTERNS.items():
|
|
||||||
result[key] = find_float(log_text, patterns)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def percentile(values: list[float], p: float) -> float:
|
|
||||||
if not values:
|
|
||||||
return 0.0
|
|
||||||
sorted_values = sorted(values)
|
|
||||||
k = (len(sorted_values) - 1) * p / 100.0
|
|
||||||
f = int(k)
|
|
||||||
c = min(f + 1, len(sorted_values) - 1)
|
|
||||||
if f == c:
|
|
||||||
return sorted_values[f]
|
|
||||||
return sorted_values[f] * (c - k) + sorted_values[c] * (k - f)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_jsonl(jsonl_path: Path) -> tuple[list[dict], dict]:
|
|
||||||
"""Parse sglang.bench_serving --output-file output.
|
|
||||||
|
|
||||||
Newer sglang writes one aggregate JSON object. Older versions write one
|
|
||||||
JSON object per request. Returns (raw_requests, aggregates).
|
|
||||||
"""
|
|
||||||
text = jsonl_path.read_text(encoding="utf-8", errors="replace").strip()
|
|
||||||
if not text:
|
|
||||||
return [], {}
|
|
||||||
|
|
||||||
# Try single aggregate JSON object first.
|
|
||||||
try:
|
|
||||||
data = json.loads(text)
|
|
||||||
if isinstance(data, dict) and "mean_e2e_latency_ms" in data:
|
|
||||||
aggregates = {
|
|
||||||
"success": data.get("total_output_tokens", 0) > 0 and 1 or 0,
|
|
||||||
"failed": 0,
|
|
||||||
"input_tokens": data.get("total_input_tokens", 0),
|
|
||||||
"output_tokens": data.get("total_output_tokens", 0),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": {
|
|
||||||
"mean": data.get("mean_e2e_latency_ms"),
|
|
||||||
"p50": data.get("median_e2e_latency_ms"),
|
|
||||||
"p90": data.get("p90_e2e_latency_ms"),
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_e2e_latency_ms"),
|
|
||||||
},
|
|
||||||
"ttft_ms": {
|
|
||||||
"mean": data.get("mean_ttft_ms"),
|
|
||||||
"p50": data.get("median_ttft_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_ttft_ms"),
|
|
||||||
},
|
|
||||||
"tpot_ms": {
|
|
||||||
"mean": data.get("mean_tpot_ms"),
|
|
||||||
"p50": data.get("median_tpot_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": None,
|
|
||||||
"p99": data.get("p99_tpot_ms"),
|
|
||||||
},
|
|
||||||
"itl_ms": {
|
|
||||||
"mean": data.get("mean_itl_ms"),
|
|
||||||
"p50": data.get("median_itl_ms"),
|
|
||||||
"p90": None,
|
|
||||||
"p95": data.get("p95_itl_ms"),
|
|
||||||
"p99": data.get("p99_itl_ms"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
return [data], aggregates
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Fall back to JSONL per-request parsing.
|
|
||||||
raw_requests = []
|
|
||||||
ttfts = []
|
|
||||||
tpots = []
|
|
||||||
itls = []
|
|
||||||
e2es = []
|
|
||||||
input_tokens = []
|
|
||||||
output_tokens = []
|
|
||||||
success = 0
|
|
||||||
failed = 0
|
|
||||||
|
|
||||||
for line in text.splitlines():
|
|
||||||
line = line.strip()
|
|
||||||
if not line:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
req = json.loads(line)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_requests.append(req)
|
|
||||||
|
|
||||||
ttft = req.get("ttft") or req.get("ttft_ms") or 0
|
|
||||||
tpot = req.get("tpot") or req.get("tpot_ms") or 0
|
|
||||||
itl = req.get("inter_token_latency") or req.get("itl") or req.get("itl_ms") or 0
|
|
||||||
e2e = req.get("e2e_latency") or req.get("e2e") or req.get("e2e_ms") or 0
|
|
||||||
in_tok = req.get("input_tokens") or req.get("prompt_tokens") or 0
|
|
||||||
out_tok = req.get("output_tokens") or req.get("completion_tokens") or 0
|
|
||||||
|
|
||||||
if ttft:
|
|
||||||
ttfts.append(float(ttft))
|
|
||||||
if tpot:
|
|
||||||
tpots.append(float(tpot))
|
|
||||||
if itl:
|
|
||||||
itls.append(float(itl))
|
|
||||||
if e2e:
|
|
||||||
e2es.append(float(e2e))
|
|
||||||
if in_tok:
|
|
||||||
input_tokens.append(int(in_tok))
|
|
||||||
if out_tok:
|
|
||||||
output_tokens.append(int(out_tok))
|
|
||||||
|
|
||||||
if req.get("success", True):
|
|
||||||
success += 1
|
|
||||||
else:
|
|
||||||
failed += 1
|
|
||||||
|
|
||||||
def latency_bucket(values: list[float]) -> dict:
|
|
||||||
if not values:
|
|
||||||
return {"mean": None, "p50": None, "p90": None, "p95": None, "p99": None}
|
|
||||||
return {
|
|
||||||
"mean": round(sum(values) / len(values), 2),
|
|
||||||
"p50": round(percentile(values, 50), 2),
|
|
||||||
"p90": round(percentile(values, 90), 2),
|
|
||||||
"p95": round(percentile(values, 95), 2),
|
|
||||||
"p99": round(percentile(values, 99), 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
aggregates = {
|
|
||||||
"success": success,
|
|
||||||
"failed": failed,
|
|
||||||
"input_tokens": sum(input_tokens),
|
|
||||||
"output_tokens": sum(output_tokens),
|
|
||||||
"latencies": {
|
|
||||||
"e2e_ms": latency_bucket(e2es),
|
|
||||||
"ttft_ms": latency_bucket(ttfts),
|
|
||||||
"tpot_ms": latency_bucket(tpots),
|
|
||||||
"itl_ms": latency_bucket(itls),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
return raw_requests, aggregates
|
|
||||||
|
|
||||||
|
|
||||||
def parse_scenario(result_root: Path, raw_file: Path) -> dict | None:
|
|
||||||
"""Parse one raw output file into a scenario dict."""
|
|
||||||
# Filename: {chip}_{engine}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl
|
|
||||||
parts = raw_file.stem.split("_")
|
|
||||||
if len(parts) < 6:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
concurrency = int(parts[-3])
|
|
||||||
input_len = int(parts[-2])
|
|
||||||
output_len = int(parts[-1])
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
log_file = result_root / "logs" / f"sglang_c{concurrency}_i{input_len}_o{output_len}.log"
|
|
||||||
summary = {}
|
|
||||||
if log_file.exists():
|
|
||||||
summary = parse_summary_log(log_file.read_text(encoding="utf-8", errors="replace"))
|
|
||||||
|
|
||||||
raw_requests, aggregates = parse_jsonl(raw_file)
|
|
||||||
|
|
||||||
# For newer sglang aggregate JSON, success/failed are not present in the
|
|
||||||
# raw output file. Override them from the human-readable summary log when
|
|
||||||
# it is available.
|
|
||||||
if summary.get("successful_requests") is not None:
|
|
||||||
aggregates["success"] = int(summary["successful_requests"])
|
|
||||||
# The summary log only reports successes; assume failures are zero
|
|
||||||
# unless the aggregate JSON already provided a non-zero failed count.
|
|
||||||
if not aggregates.get("failed"):
|
|
||||||
aggregates["failed"] = 0
|
|
||||||
|
|
||||||
scenario = {
|
|
||||||
"name": f"c{concurrency}_i{input_len}_o{output_len}",
|
|
||||||
"concurrency": concurrency,
|
|
||||||
"input_len": input_len,
|
|
||||||
"output_len": output_len,
|
|
||||||
"success": aggregates["success"],
|
|
||||||
"failed": aggregates["failed"],
|
|
||||||
"duration_s": summary.get("benchmark_duration_s"),
|
|
||||||
"request_throughput": summary.get("request_throughput"),
|
|
||||||
"input_token_throughput": summary.get("input_token_throughput"),
|
|
||||||
"output_token_throughput": summary.get("output_token_throughput"),
|
|
||||||
"total_token_throughput": summary.get("total_token_throughput"),
|
|
||||||
"accept_length": None,
|
|
||||||
"latencies": aggregates["latencies"],
|
|
||||||
"raw_requests": raw_requests[:100] if len(raw_requests) <= 100 else None,
|
|
||||||
}
|
|
||||||
|
|
||||||
return scenario
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
result_root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("results")
|
|
||||||
raw_dir = result_root / "raw_outputs"
|
|
||||||
json_path = result_root / "results.json"
|
|
||||||
report_path = result_root / "report.md"
|
|
||||||
|
|
||||||
if not json_path.exists():
|
|
||||||
raise SystemExit(f"metadata results.json not found: {json_path}")
|
|
||||||
|
|
||||||
with open(json_path, "r", encoding="utf-8") as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
scenarios = []
|
|
||||||
if raw_dir.exists():
|
|
||||||
for raw_file in sorted(raw_dir.glob("*.jsonl")):
|
|
||||||
scenario = parse_scenario(result_root, raw_file)
|
|
||||||
if scenario:
|
|
||||||
scenarios.append(scenario)
|
|
||||||
|
|
||||||
data["scenarios"] = scenarios
|
|
||||||
|
|
||||||
with open(json_path, "w", encoding="utf-8") as f:
|
|
||||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
||||||
|
|
||||||
# Generate report.md
|
|
||||||
with open(report_path, "w", encoding="utf-8") as f:
|
|
||||||
meta = data["metadata"]
|
|
||||||
f.write(f"# Benchmark Report: {meta['experiment']}\n\n")
|
|
||||||
f.write("## Metadata\n\n")
|
|
||||||
f.write(f"- **Run ID**: {meta['run_id']}\n")
|
|
||||||
f.write(f"- **Timestamp**: {meta['timestamp']}\n")
|
|
||||||
f.write(f"- **Chip/Accelerator**: {meta.get('accelerator', '')} / {meta.get('chip', '')}\n")
|
|
||||||
f.write(f"- **Engine/Backend**: {meta.get('engine', '')} / {meta.get('backend', '')}\n")
|
|
||||||
f.write(f"- **Hardware**: {meta.get('hardware', '')}\n")
|
|
||||||
f.write(f"- **Model**: {meta['model']}\n")
|
|
||||||
f.write(f"- **Git Commit**: {meta.get('git_commit', 'unknown')}\n")
|
|
||||||
f.write("\n## Results\n\n")
|
|
||||||
f.write(
|
|
||||||
"| Scenario | Conc | In/Out | Success | Failed | Req/s | OutTok/s | "
|
|
||||||
"TTFT p50 | TTFT p99 | TPOT p50 | TPOT p99 | E2E p99 |\n"
|
|
||||||
)
|
|
||||||
f.write(
|
|
||||||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n"
|
|
||||||
)
|
|
||||||
for s in scenarios:
|
|
||||||
lat = s["latencies"]
|
|
||||||
f.write(
|
|
||||||
f"| {s['name']} "
|
|
||||||
f"| {s['concurrency']} "
|
|
||||||
f"| {s['input_len']}/{s['output_len']} "
|
|
||||||
f"| {s['success']} "
|
|
||||||
f"| {s['failed']} "
|
|
||||||
f"| {s.get('request_throughput') or ''} "
|
|
||||||
f"| {s.get('output_token_throughput') or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['ttft_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p50'] or ''} "
|
|
||||||
f"| {lat['tpot_ms']['p99'] or ''} "
|
|
||||||
f"| {lat['e2e_ms']['p99'] or ''} |\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
print(f"Updated {json_path}")
|
|
||||||
print(f"Wrote {report_path}")
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
28
pyproject.toml
Normal file
28
pyproject.toml
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=61.0"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "sskj-bench"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Multi-platform LLM serving benchmark framework (sglang/vllm on P800/H20/H200/RTX 6000D)"
|
||||||
|
requires-python = ">=3.10"
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"ruff>=0.5.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 120
|
||||||
|
target-version = "py310"
|
||||||
|
extend-exclude = ["envs", "experiments/*/results", "experiments/*/runtime", ".tmp_charts"]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4"]
|
||||||
|
ignore = ["E501"]
|
||||||
|
|
||||||
|
[tool.ruff.format]
|
||||||
|
quote-style = "double"
|
||||||
|
indent-style = "space"
|
||||||
3
requirements-dev.txt
Normal file
3
requirements-dev.txt
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
# Dev tooling for sskj benchmark repo.
|
||||||
|
# Runtime Python code uses only the standard library (no runtime deps).
|
||||||
|
ruff>=0.5.0
|
||||||
@ -63,17 +63,18 @@ health_check() {
|
|||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
|
|
||||||
git_commit() {
|
git_commit() {
|
||||||
cd "$ROOT_DIR" || return 1
|
( cd "$ROOT_DIR" && git rev-parse --short HEAD 2>/dev/null || echo "unknown" )
|
||||||
git rev-parse --short HEAD 2>/dev/null || echo "unknown"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
git_dirty() {
|
git_dirty() {
|
||||||
cd "$ROOT_DIR" || return 1
|
(
|
||||||
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
|
cd "$ROOT_DIR" || exit 1
|
||||||
echo "dirty"
|
if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
|
||||||
else
|
echo "dirty"
|
||||||
echo "clean"
|
else
|
||||||
fi
|
echo "clean"
|
||||||
|
fi
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
@ -96,28 +97,34 @@ write_metadata_json() {
|
|||||||
|
|
||||||
mkdir -p "$(dirname "$output_path")"
|
mkdir -p "$(dirname "$output_path")"
|
||||||
|
|
||||||
cat > "$output_path" <<EOF
|
"${PYTHON:-python3}" - "$output_path" "$experiment" "$run_id" "$model" "$backend" "$engine" "$hardware" "$accelerator" "$chip" "$script" "$env_path" "$description" "$(date --iso-8601=seconds)" "$(git_commit)" "$(git_dirty)" <<'PY'
|
||||||
{
|
import json, sys
|
||||||
"metadata": {
|
(out, experiment, run_id, model, backend, engine, hardware, accelerator,
|
||||||
"experiment": "${experiment}",
|
chip, script, env_path, description, timestamp, git_commit,
|
||||||
"run_id": "${run_id}",
|
git_dirty) = sys.argv[1:16]
|
||||||
"timestamp": "$(date --iso-8601=seconds)",
|
data = {
|
||||||
"model": "${model}",
|
"metadata": {
|
||||||
"backend": "${backend}",
|
"experiment": experiment,
|
||||||
"engine": "${engine}",
|
"run_id": run_id,
|
||||||
"hardware": "${hardware}",
|
"timestamp": timestamp,
|
||||||
"accelerator": "${accelerator}",
|
"model": model,
|
||||||
"chip": "${chip}",
|
"backend": backend,
|
||||||
"script": "${script}",
|
"engine": engine,
|
||||||
"env": "${env_path}",
|
"hardware": hardware,
|
||||||
"git_commit": "$(git_commit)",
|
"accelerator": accelerator,
|
||||||
"git_dirty": "$(git_dirty)",
|
"chip": chip,
|
||||||
"description": "${description}"
|
"script": script,
|
||||||
},
|
"env": env_path,
|
||||||
"config": {},
|
"git_commit": git_commit,
|
||||||
"scenarios": []
|
"git_dirty": git_dirty,
|
||||||
|
"description": description,
|
||||||
|
},
|
||||||
|
"config": {},
|
||||||
|
"scenarios": [],
|
||||||
}
|
}
|
||||||
EOF
|
with open(out, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||||
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user