Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run Summary: tok in/out columns with per-direction throughput (in/s, out/s), totals; clickable file:// links in artifacts notice; drop note column

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 09:59:02 +00:00
parent 82d9d03b88
commit b4d39c560e
1 changed files with 35 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
evalharness/cli.py
View File

@ -510,9 +510,10 @@ def _cmd_eval_run(args) -> int:
'secs': round(secs_total, 1), 'secs': round(secs_total, 1),
'wall': round(_time.time() - t0, 1), 'wall': round(_time.time() - t0, 1),
'hours': round(secs_total / 3600, 2), 'hours': round(secs_total / 3600, 2),
'tok_in': info.get('gen_input_tokens', 0) or 0,
'tok_out': info.get('gen_output_tokens', 0) or 0,
'tokens': (info.get('gen_input_tokens', 0) or 0) 'tokens': (info.get('gen_input_tokens', 0) or 0)
+ (info.get('gen_output_tokens', 0) or 0), + (info.get('gen_output_tokens', 0) or 0),
'tok_out': info.get('gen_output_tokens', 0) or 0,
'groups': groups, 'ok': True}) 'groups': groups, 'ok': True})
_print_benchmark_result(console, i + 1, total_runs, name, _print_benchmark_result(console, i + 1, total_runs, name,
'done', _time.time() - t0) 'done', _time.time() - t0)
@ -574,17 +575,30 @@ def _cmd_eval_run(args) -> int:
f"{r.get('n', '')} | {t} | {st} |\n") f"{r.get('n', '')} | {t} | {st} |\n")
if out_dir: if out_dir:
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)') print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
# artifacts notice: tell the user where everything landed (or how to save) # artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)
def _notice(label, *paths):
if console is not None:
parts = []
for p in paths:
ap = os.path.abspath(p)
parts.append(f'[link=file://{ap}]{ap}[/link]')
console.print(f'\n[bold]{label}[/bold] -> ' + ' · '.join(parts))
else:
print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths))
if len(rows) > 1: if len(rows) > 1:
if out_dir: if out_dir:
print(f'\n结果已生成 -> {out_dir}/reports/<bench>.report.json · ' _notice('结果已生成', f'{out_dir}/reports/<bench>.report.json',
f'{out_dir}/viz/summary.md') f'{out_dir}/viz/summary.md')
else: else:
print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)') print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)')
elif rows and rows[0]['ok']: elif rows and rows[0]['ok']:
saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '') saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '')
print(f'\n结果已生成 -> {saved}' if saved else if saved:
'\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告') _notice('结果已生成', saved)
else:
print('\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告')
return 0 if all(r['ok'] for r in rows) else 1 return 0 if all(r['ok'] for r in rows) else 1
@ -606,31 +620,33 @@ def _print_summary_table(console, rows):
title_style='bold', expand=False) title_style='bold', expand=False)
for col, just in (('benchmark', 'left'), ('metric', 'left'), for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'), ('score', 'right'), ('n', 'right'), ('time', 'right'),
('tokens', 'right'), ('tok/s', 'right'), ('note', 'left')): ('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right')):
t.add_column(col, justify=just) t.add_column(col, justify=just)
for r in rows: for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
wall = r.get('wall') or r.get('secs') or 0 wall = r.get('wall') or r.get('secs') or 0
tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s' tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s'
tok = f"{r.get('tokens', 0):,}" if r.get('tokens') else '—' ti, to = r.get('tok_in', 0), r.get('tok_out', 0)
tps = (r.get('tok_out', 0) / wall) if (wall > 1 and r.get('tok_out')) else 0 tin = f'{ti:,}' if ti else '—'
tp = f'{tps:.0f}' if tps else '—' tout = f'{to:,}' if to else '—'
if not r['ok']: tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
note = f"[red]{str(r.get('err', ''))[:34]}[/red]" tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
elif r.get('extract_fail'): t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
note = f"[yellow]extract-fail {r['extract_fail']}[/yellow]" tin, tout, tis, tos,
else:
note = ''
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, tok, tp, note,
style='green' if r['ok'] else 'red') style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
tok_all = sum(r.get('tokens', 0) for r in rows) ti_all = sum(r.get('tok_in', 0) for r in rows)
to_all = sum(r.get('tok_out', 0) for r in rows)
n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int)) n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int))
tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s' tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s'
t.add_section() t.add_section()
tis_all = f'{ti_all / wall_all:.0f}' if wall_all > 1 else ''
tos_all = f'{to_all / wall_all:.0f}' if wall_all > 1 else ''
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{tok_all:,}', '', '') str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all)
console.print(t) console.print(t)
return return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}') print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')