Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify closing block to one clean status+directory tree; drop lat/trunc columns from summary

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:08:21 +00:00
parent ed36fda365
commit 3a1bfb9fde
1 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
evalharness/cli.py
View File

@ -551,8 +551,6 @@ def _cmd_eval_run(args) -> int:
if len(rows) > 1:
_print_summary_table(console, rows)
ok = sum(1 for r in rows if r['ok'])
print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else ''))
if out_dir:
import csv as _csv
@ -597,8 +595,7 @@ def _cmd_eval_run(args) -> int:
st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}"
f.write(f"| {r['name']} | {r['metric']} | {v} | "
f"{r.get('n', '')} | {t} | {st} |\n")
if out_dir:
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
# artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)
def _notice(label, *paths):
@ -613,11 +610,20 @@ def _cmd_eval_run(args) -> int:
ok_n = sum(1 for r in rows if r['ok'])
if out_dir:
_notice('运行结束 · 结果已保存', f'{out_dir}/reports/<bench>.report.json',
f'{out_dir}/viz/summary.md')
ap = os.path.abspath(out_dir)
if console is not None:
mark = '[green]✓[/green]' if ok_n == len(rows) else '[yellow]◐[/yellow]'
console.print(
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 [link=file://{ap}]{ap}[/link]\n'
f' ├─ [link=file://{ap}/reports]reports/<bench>.report.json[/link]\n'
f' └─ [link=file://{ap}/viz/summary.md]viz/summary.md[/link] (+ summary.csv)')
else:
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md')
elif rows and rows[0]['ok'] and args.out:
_notice('运行结束 · 结果已保存', args.out)
print(f'\n{ok_n}/{len(rows)} benchmarks ok')
print(f'{ok_n}/{len(rows)} benchmarks ok')
return 0 if all(r['ok'] for r in rows) else 1
@ -640,8 +646,7 @@ def _print_summary_table(console, rows):
for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'),
('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right'),
('lat p50/p90', 'right'), ('trunc', 'right')):
('in/s', 'right'), ('out/s', 'right')):
t.add_column(col, justify=just)
for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
@ -652,11 +657,8 @@ def _print_summary_table(console, rows):
tout = f'{to:,}' if to else '—'
tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
lat = f"{r.get('lat_p50', 0):.1f}/{r.get('lat_p90', 0):.1f}s" \
if r.get('lat_p50') else '—'
trunc = str(r['trunc']) if r.get('trunc') else '—'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
tin, tout, tis, tos, lat, trunc,
tin, tout, tis, tos,
style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
ti_all = sum(r.get('tok_in', 0) for r in rows)
@ -669,7 +671,7 @@ def _print_summary_table(console, rows):
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all, '', str(sum(r.get('trunc', 0) for r in rows)))
tis_all, tos_all)
console.print(t, justify='center')
return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')