Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run Summary: tok in/out columns with per-direction throughput (in/s, out/s), totals; clickable file:// links in artifacts notice; drop note column

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 09:59:02 +00:00
parent 82d9d03b88
commit b4d39c560e
1 changed files with 35 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
evalharness/cli.py
View File

@ -510,9 +510,10 @@ def _cmd_eval_run(args) -> int:
'secs': round(secs_total, 1),
'wall': round(_time.time() - t0, 1),
'hours': round(secs_total / 3600, 2),
'tok_in': info.get('gen_input_tokens', 0) or 0,
'tok_out': info.get('gen_output_tokens', 0) or 0,
'tokens': (info.get('gen_input_tokens', 0) or 0)
+ (info.get('gen_output_tokens', 0) or 0),
'tok_out': info.get('gen_output_tokens', 0) or 0,
'groups': groups, 'ok': True})
_print_benchmark_result(console, i + 1, total_runs, name,
'done', _time.time() - t0)
@ -574,17 +575,30 @@ def _cmd_eval_run(args) -> int:
f"{r.get('n', '')} | {t} | {st} |\n")
if out_dir:
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
# artifacts notice: tell the user where everything landed (or how to save)
# artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)
def _notice(label, *paths):
if console is not None:
parts = []
for p in paths:
ap = os.path.abspath(p)
parts.append(f'[link=file://{ap}]{ap}[/link]')
console.print(f'\n[bold]{label}[/bold] -> ' + ' · '.join(parts))
else:
print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths))
if len(rows) > 1:
if out_dir:
print(f'\n结果已生成 -> {out_dir}/reports/<bench>.report.json · '
_notice('结果已生成', f'{out_dir}/reports/<bench>.report.json',
f'{out_dir}/viz/summary.md')
else:
print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)')
elif rows and rows[0]['ok']:
saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '')
print(f'\n结果已生成 -> {saved}' if saved else
'\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告')
if saved:
_notice('结果已生成', saved)
else:
print('\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告')
return 0 if all(r['ok'] for r in rows) else 1
@ -606,31 +620,33 @@ def _print_summary_table(console, rows):
title_style='bold', expand=False)
for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'),
('tokens', 'right'), ('tok/s', 'right'), ('note', 'left')):
('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right')):
t.add_column(col, justify=just)
for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
wall = r.get('wall') or r.get('secs') or 0
tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s'
tok = f"{r.get('tokens', 0):,}" if r.get('tokens') else '—'
tps = (r.get('tok_out', 0) / wall) if (wall > 1 and r.get('tok_out')) else 0
tp = f'{tps:.0f}' if tps else '—'
if not r['ok']:
note = f"[red]{str(r.get('err', ''))[:34]}[/red]"
elif r.get('extract_fail'):
note = f"[yellow]extract-fail {r['extract_fail']}[/yellow]"
else:
note = ''
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, tok, tp, note,
ti, to = r.get('tok_in', 0), r.get('tok_out', 0)
tin = f'{ti:,}' if ti else '—'
tout = f'{to:,}' if to else '—'
tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
tin, tout, tis, tos,
style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
tok_all = sum(r.get('tokens', 0) for r in rows)
ti_all = sum(r.get('tok_in', 0) for r in rows)
to_all = sum(r.get('tok_out', 0) for r in rows)
n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int))
tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s'
t.add_section()
tis_all = f'{ti_all / wall_all:.0f}' if wall_all > 1 else ''
tos_all = f'{to_all / wall_all:.0f}' if wall_all > 1 else ''
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{tok_all:,}', '', '')
str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all)
console.print(t)
return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')