Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify closing block to one clean status+directory tree; drop lat/trunc columns from summary

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:08:21 +00:00
parent ed36fda365
commit 3a1bfb9fde
1 changed files with 16 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
evalharness/cli.py
View File

@ -551,8 +551,6 @@ def _cmd_eval_run(args) -> int:
if len(rows) > 1: if len(rows) > 1:
_print_summary_table(console, rows) _print_summary_table(console, rows)
ok = sum(1 for r in rows if r['ok'])
print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else ''))
if out_dir: if out_dir:
import csv as _csv import csv as _csv
@ -597,8 +595,7 @@ def _cmd_eval_run(args) -> int:
st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}" st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}"
f.write(f"| {r['name']} | {r['metric']} | {v} | " f.write(f"| {r['name']} | {r['metric']} | {v} | "
f"{r.get('n', '')} | {t} | {st} |\n") f"{r.get('n', '')} | {t} | {st} |\n")
if out_dir:
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
# artifacts notice: tell the user where everything landed (or how to save); # artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...) # rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)
def _notice(label, *paths): def _notice(label, *paths):
@ -613,11 +610,20 @@ def _cmd_eval_run(args) -> int:
ok_n = sum(1 for r in rows if r['ok']) ok_n = sum(1 for r in rows if r['ok'])
if out_dir: if out_dir:
_notice('运行结束 · 结果已保存', f'{out_dir}/reports/<bench>.report.json', ap = os.path.abspath(out_dir)
f'{out_dir}/viz/summary.md') if console is not None:
mark = '[green]✓[/green]' if ok_n == len(rows) else '[yellow]◐[/yellow]'
console.print(
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 [link=file://{ap}]{ap}[/link]\n'
f' ├─ [link=file://{ap}/reports]reports/<bench>.report.json[/link]\n'
f' └─ [link=file://{ap}/viz/summary.md]viz/summary.md[/link] (+ summary.csv)')
else:
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md')
elif rows and rows[0]['ok'] and args.out: elif rows and rows[0]['ok'] and args.out:
_notice('运行结束 · 结果已保存', args.out) _notice('运行结束 · 结果已保存', args.out)
print(f'\n{ok_n}/{len(rows)} benchmarks ok') print(f'{ok_n}/{len(rows)} benchmarks ok')
return 0 if all(r['ok'] for r in rows) else 1 return 0 if all(r['ok'] for r in rows) else 1
@ -640,8 +646,7 @@ def _print_summary_table(console, rows):
for col, just in (('benchmark', 'left'), ('metric', 'left'), for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'), ('score', 'right'), ('n', 'right'), ('time', 'right'),
('tok in', 'right'), ('tok out', 'right'), ('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right'), ('in/s', 'right'), ('out/s', 'right')):
('lat p50/p90', 'right'), ('trunc', 'right')):
t.add_column(col, justify=just) t.add_column(col, justify=just)
for r in rows: for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
@ -652,11 +657,8 @@ def _print_summary_table(console, rows):
tout = f'{to:,}' if to else '—' tout = f'{to:,}' if to else '—'
tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—' tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—' tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
lat = f"{r.get('lat_p50', 0):.1f}/{r.get('lat_p90', 0):.1f}s" \
if r.get('lat_p50') else '—'
trunc = str(r['trunc']) if r.get('trunc') else '—'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
tin, tout, tis, tos, lat, trunc, tin, tout, tis, tos,
style='green' if r['ok'] else 'red') style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
ti_all = sum(r.get('tok_in', 0) for r in rows) ti_all = sum(r.get('tok_in', 0) for r in rows)
@ -669,7 +671,7 @@ def _print_summary_table(console, rows):
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}', str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all, '', str(sum(r.get('trunc', 0) for r in rows))) tis_all, tos_all)
console.print(t, justify='center') console.print(t, justify='center')
return return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}') print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')