Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Incremental summary: flush summary.csv/xlsx after every benchmark

Browse Source 
Previously both were written only after ALL benches finished -- a
multi-hour mmlu_pro left the summary stale for hours, and a crashed
run left the PREVIOUS run's files in place. Now each completed bench
(including failed ones) rewrites the summaries with everything so far.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 05:56:42 +00:00
parent 0d925fc46f
commit 97c4eef8fb
1 changed files with 53 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
evalharness/cli.py
View File

@ -621,6 +621,54 @@ def _cmd_eval_run(args) -> int:
rows = []
all_reports = []
def _flush_summaries():
"""Rewrite summary.csv/xlsx from the benches finished SO FAR.
Called after every benchmark instead of only at the very end:
multi-hour runs (mmlu_pro 12k samples) leave the summary stale for
hours otherwise -- and a crashed run would leave the LAST run's
corpses in place instead of partial results."""
if not out_dir:
return
if all_reports:
try:
from evalharness.viz import render as _render
_render(all_reports, style='excel', out=f'{out_dir}/summary.xlsx')
except Exception as e:
print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}',
file=sys.stderr)
if not rows:
return
import csv as _csv
with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f:
w = _csv.writer(f)
w.writerow(['benchmark', 'score', 'metric', 'num_samples',
'time_h', 'time_s', 'extract_fail',
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s',
'categories'])
for r in rows:
perf = (r.get('groups') or {}).get('perf') or {}
cats = '; '.join(f'{g}={_f3(v)}'
for gname, gv in (r.get('groups') or {}).items()
if gname != 'perf'
for g, v in (gv or {}).items()
if isinstance(v, (int, float)))[:2000]
w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''),
r.get('hours', ''), r.get('secs', ''),
r.get('extract_fail', 0)] +
[perf.get(k, '') for k in (
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] +
[cats])
run_started = _time.time()
total_runs = len(args.datasets)
model_spec = _compose_model_spec(args)
@ -908,49 +956,17 @@ def _cmd_eval_run(args) -> int:
border_style='red', expand=False), justify='center')
_print_benchmark_result(console, i + 1, total_runs, name,
'failed', _time.time() - t0)
# incremental summary: partial results are visible (and survive a
# crash) after EVERY benchmark, not only when the whole run ends
_flush_summaries()
if _shared_reporter is not None:
_shared_reporter.close()
if all_reports and out_dir:
try:
from evalharness.viz import render as _render
xb = _render(all_reports, style='excel',
out=f'{out_dir}/summary.xlsx')
print(f'excel -> {xb}', flush=True)
except Exception as e:
print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}',
file=sys.stderr)
_flush_summaries() # final state (benches already flushed per-bench)
print(f'excel -> {out_dir}/summary.xlsx', flush=True)
if rows:
_print_summary_table(console, rows)
if out_dir:
import csv as _csv
with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f:
w = _csv.writer(f)
w.writerow(['benchmark', 'score', 'metric', 'num_samples',
'time_h', 'time_s', 'extract_fail',
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s',
'categories'])
for r in rows:
perf = (r.get('groups') or {}).get('perf') or {}
cats = '; '.join(f'{g}={_f3(v)}'
for gname, gv in (r.get('groups') or {}).items()
if gname != 'perf'
for g, v in (gv or {}).items()
if isinstance(v, (int, float)))[:2000]
w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''),
r.get('hours', ''), r.get('secs', ''),
r.get('extract_fail', 0)] +
[perf.get(k, '') for k in (
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] +
[cats])
# artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)