Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run Summary: tokens/throughput/note columns + totals row; final artifacts notice (where results were saved, or how to save)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 09:49:36 +00:00
parent 075e8728cb
commit 82d9d03b88
1 changed files with 39 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
evalharness/cli.py
View File

@ -503,11 +503,16 @@ def _cmd_eval_run(args) -> int:
groups = {k: v for k, v in report.metric_groups.items()
if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')}
info = report.metric_groups.get('run_info', {}) or {}
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
'n': report.num_samples,
'extract_fail': report.num_failed_extractions,
'secs': round(secs_total, 1),
'wall': round(_time.time() - t0, 1),
'hours': round(secs_total / 3600, 2),
'tokens': (info.get('gen_input_tokens', 0) or 0)
+ (info.get('gen_output_tokens', 0) or 0),
'tok_out': info.get('gen_output_tokens', 0) or 0,
'groups': groups, 'ok': True})
_print_benchmark_result(console, i + 1, total_runs, name,
'done', _time.time() - t0)
@ -569,6 +574,17 @@ def _cmd_eval_run(args) -> int:
f"{r.get('n', '')} | {t} | {st} |\n")
if out_dir:
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
# artifacts notice: tell the user where everything landed (or how to save)
if len(rows) > 1:
if out_dir:
print(f'\n结果已生成 -> {out_dir}/reports/<bench>.report.json · '
f'{out_dir}/viz/summary.md')
else:
print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)')
elif rows and rows[0]['ok']:
saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '')
print(f'\n结果已生成 -> {saved}' if saved else
'\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告')
return 0 if all(r['ok'] for r in rows) else 1
@ -589,15 +605,32 @@ def _print_summary_table(console, rows):
t = Table(title='Run Summary', header_style='bold cyan',
title_style='bold', expand=False)
for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right')):
('score', 'right'), ('n', 'right'), ('time', 'right'),
('tokens', 'right'), ('tok/s', 'right'), ('note', 'left')):
t.add_column(col, justify=just)
for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
h = r.get('hours') or 0
tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s"
style = 'green' if r['ok'] else 'red'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
style=style)
wall = r.get('wall') or r.get('secs') or 0
tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s'
tok = f"{r.get('tokens', 0):,}" if r.get('tokens') else '—'
tps = (r.get('tok_out', 0) / wall) if (wall > 1 and r.get('tok_out')) else 0
tp = f'{tps:.0f}' if tps else '—'
if not r['ok']:
note = f"[red]{str(r.get('err', ''))[:34]}[/red]"
elif r.get('extract_fail'):
note = f"[yellow]extract-fail {r['extract_fail']}[/yellow]"
else:
note = ''
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, tok, tp, note,
style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
tok_all = sum(r.get('tokens', 0) for r in rows)
n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int))
tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s'
t.add_section()
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{tok_all:,}', '', '')
console.print(t)
return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')