From e77ea4ce5cb7bce28f4fd5d0e48bd1cff5713f03 Mon Sep 17 00:00:00 2001 From: sora Date: Thu, 27 Aug 2026 02:49:55 +0000 Subject: [PATCH] !perf spec flag; perf_stats full dashboard columns (success_rate, latency/ttft/tpot mean+P90+P99, tokens mean/total, output TPS, request QPS); summary.csv perf columns aligned --- evalharness/cli.py | 17 ++++++++++++++-- evalharness/eval/aggregator.py | 37 +++++++++++++++++++++++++--------- evalharness/model/runner.py | 4 +++- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 77faf51..90422c8 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -205,15 +205,28 @@ def _cmd_eval_run(args) -> int: with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f: w = _csv.writer(f) w.writerow(['benchmark', 'score', 'metric', 'num_samples', - 'time_h', 'time_s', 'extract_fail', 'categories']) + 'time_h', 'time_s', 'extract_fail', + 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', + 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', + 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', + 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s', + 'categories']) for r in rows: + perf = (r.get('groups') or {}).get('perf') or {} cats = '; '.join(f'{g}={_f3(v)}' for gname, gv in (r.get('groups') or {}).items() + if gname != 'perf' for g, v in (gv or {}).items() if isinstance(v, (int, float)))[:2000] w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''), r.get('hours', ''), r.get('secs', ''), - r.get('extract_fail', 0), cats]) + r.get('extract_fail', 0)] + + [perf.get(k, '') for k in ( + 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', + 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', + 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', + 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + + [cats]) with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f: f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n') for r in rows: diff --git a/evalharness/eval/aggregator.py b/evalharness/eval/aggregator.py index 0485cfb..d784d8f 100644 --- a/evalharness/eval/aggregator.py +++ b/evalharness/eval/aggregator.py @@ -135,19 +135,38 @@ def perf_stats(results: List[SampleResult], metric: str): if (r.usage or {}).get('ttft_s') is not None] itl = [float(r.usage['itl_mean_s']) for r in results if (r.usage or {}).get('itl_mean_s') is not None] - in_tok = sum(int((r.usage or {}).get('input_tokens', 0) or 0) for r in results) - out_tok = sum(int((r.usage or {}).get('output_tokens', 0) or 0) for r in results) + n = len(lat) + in_toks = [int((r.usage or {}).get('input_tokens', 0) or 0) for r in results] + out_toks = [int((r.usage or {}).get('output_tokens', 0) or 0) for r in results] + in_tok, out_tok = sum(in_toks), sum(out_toks) retried = sum(1 for r in results if (r.usage or {}).get('retries')) + ok = sum(1 for r in results + if (r.usage or {}).get('http_status') in (None, 200)) # None=unmeasured wall = sum(lat) + # TPOT per request: (latency - ttft) / max(output_tokens - 1, 1) + tpots = [] + for r in results: + u = r.usage or {} + lt, tf, ot = u.get('latency_s'), u.get('ttft_s'), u.get('output_tokens') + if lt and tf is not None and ot and ot > 1: + tpots.append((lt - tf) / (ot - 1)) out = { - 'n_requests': len(lat), - 'latency_p50_s': _pct(lat, 50), 'latency_p95_s': _pct(lat, 95), - 'latency_p99_s': _pct(lat, 99), 'latency_mean_s': round(statistics.mean(lat), 3) if lat else None, - 'ttft_p50_s': _pct(ttft, 50), 'ttft_p95_s': _pct(ttft, 95), - 'itl_mean_s': round(statistics.mean(itl), 4) if itl else None, - 'input_tokens': in_tok, 'output_tokens': out_tok, - 'retry_rate': round(retried / len(lat), 3) if lat else None, + 'n_requests': n, + 'success_rate': round(ok / n, 4) if n else None, + 'latency_mean_s': round(statistics.mean(lat), 3) if lat else None, + 'latency_p50_s': _pct(lat, 50), 'latency_p90_s': _pct(lat, 90), + 'latency_p95_s': _pct(lat, 95), 'latency_p99_s': _pct(lat, 99), + 'ttft_mean_s': round(statistics.mean(ttft), 3) if ttft else None, + 'ttft_p90_s': _pct(ttft, 90), 'ttft_p99_s': _pct(ttft, 99), + 'tpot_mean_s': round(statistics.mean(tpots), 4) if tpots else None, + 'tpot_p90_s': _pct(tpots, 90), 'tpot_p99_s': _pct(tpots, 99), + 'input_tokens_mean': round(statistics.mean(in_toks), 1) if in_toks else 0, + 'output_tokens_mean': round(statistics.mean(out_toks), 1) if out_toks else 0, + 'total_tokens': in_tok + out_tok, + 'output_tps': round(out_tok / wall, 2) if wall else None, # tokens/s + 'request_qps': round(n / wall, 4) if wall else None, # req/s 'wall_latency_s': round(wall, 1), + 'retry_rate': round(retried / n, 3) if n else None, } return out diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 282f1b1..569715a 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -405,7 +405,7 @@ def _make_adapter(spec: str) -> ModelAdapter: return _CACHE[cache_key] opts = {} while True: - for f in ('!nothink', '!textools'): + for f in ('!nothink', '!textools', '!perf'): if spec.endswith(f): spec = spec[:-len(f)] opts[f] = True @@ -437,6 +437,8 @@ def _make_adapter(spec: str) -> ModelAdapter: a.extra['no_think'] = True if opts.get('!textools'): a.extra['tools_mode'] = 'text' + if opts.get('!perf'): + a.extra['collect_perf'] = True _CACHE[cache_key] = adapter return adapter