diff --git a/evalharness/cli.py b/evalharness/cli.py index 91c472e..b6e7b44 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -621,6 +621,54 @@ def _cmd_eval_run(args) -> int: rows = [] all_reports = [] + + def _flush_summaries(): + """Rewrite summary.csv/xlsx from the benches finished SO FAR. + + Called after every benchmark instead of only at the very end: + multi-hour runs (mmlu_pro 12k samples) leave the summary stale for + hours otherwise -- and a crashed run would leave the LAST run's + corpses in place instead of partial results.""" + if not out_dir: + return + if all_reports: + try: + from evalharness.viz import render as _render + + _render(all_reports, style='excel', out=f'{out_dir}/summary.xlsx') + except Exception as e: + print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}', + file=sys.stderr) + if not rows: + return + import csv as _csv + + with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f: + w = _csv.writer(f) + w.writerow(['benchmark', 'score', 'metric', 'num_samples', + 'time_h', 'time_s', 'extract_fail', + 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', + 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', + 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', + 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s', + 'categories']) + for r in rows: + perf = (r.get('groups') or {}).get('perf') or {} + cats = '; '.join(f'{g}={_f3(v)}' + for gname, gv in (r.get('groups') or {}).items() + if gname != 'perf' + for g, v in (gv or {}).items() + if isinstance(v, (int, float)))[:2000] + w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''), + r.get('hours', ''), r.get('secs', ''), + r.get('extract_fail', 0)] + + [perf.get(k, '') for k in ( + 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', + 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', + 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', + 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + + [cats]) + run_started = _time.time() total_runs = len(args.datasets) model_spec = _compose_model_spec(args) @@ -908,49 +956,17 @@ def _cmd_eval_run(args) -> int: border_style='red', expand=False), justify='center') _print_benchmark_result(console, i + 1, total_runs, name, 'failed', _time.time() - t0) + # incremental summary: partial results are visible (and survive a + # crash) after EVERY benchmark, not only when the whole run ends + _flush_summaries() if _shared_reporter is not None: _shared_reporter.close() if all_reports and out_dir: - try: - from evalharness.viz import render as _render - - xb = _render(all_reports, style='excel', - out=f'{out_dir}/summary.xlsx') - print(f'excel -> {xb}', flush=True) - except Exception as e: - print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}', - file=sys.stderr) + _flush_summaries() # final state (benches already flushed per-bench) + print(f'excel -> {out_dir}/summary.xlsx', flush=True) if rows: _print_summary_table(console, rows) - if out_dir: - import csv as _csv - - with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f: - w = _csv.writer(f) - w.writerow(['benchmark', 'score', 'metric', 'num_samples', - 'time_h', 'time_s', 'extract_fail', - 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', - 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', - 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', - 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s', - 'categories']) - for r in rows: - perf = (r.get('groups') or {}).get('perf') or {} - cats = '; '.join(f'{g}={_f3(v)}' - for gname, gv in (r.get('groups') or {}).items() - if gname != 'perf' - for g, v in (gv or {}).items() - if isinstance(v, (int, float)))[:2000] - w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''), - r.get('hours', ''), r.get('secs', ''), - r.get('extract_fail', 0)] + - [perf.get(k, '') for k in ( - 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', - 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', - 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', - 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + - [cats]) # artifacts notice: tell the user where everything landed (or how to save); # rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)