Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Single-bench runs use the unified Run Summary table (panel behind --verbose); short model names in panel titles; plain-text paths in closing block (OSC8 links unreliable across terminals)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:25:56 +00:00
parent 1ef230f4c1
commit 8cd45e4d16
1 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
evalharness/cli.py
View File

@ -266,7 +266,12 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0 tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0
tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0 tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0
title = f'{rep.dataset} · {rep.model or "?"}' import re as _re
m = str(rep.model or '?').split('?')[-1]
m = _re.sub(r'![a-z]+$', '', m).strip('/')
m = m.rsplit('/', 1)[-1] if '/' in m else m
title = f'{rep.dataset} · {m}'
body = Table(show_header=False, box=None, padding=(0, 2)) body = Table(show_header=False, box=None, padding=(0, 2))
body.add_column('k', style='dim', no_wrap=True) body.add_column('k', style='dim', no_wrap=True)
body.add_column('v', overflow='fold') body.add_column('v', overflow='fold')
@ -355,7 +360,12 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0 tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0
tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0 tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0
title = f'{rep.dataset} · {rep.model or "?"}' import re as _re
m = str(rep.model or '?').split('?')[-1]
m = _re.sub(r'![a-z]+$', '', m).strip('/')
m = m.rsplit('/', 1)[-1] if '/' in m else m
title = f'{rep.dataset} · {m}'
body = Table(show_header=False, box=None, padding=(0, 2)) body = Table(show_header=False, box=None, padding=(0, 2))
body.add_column('k', style='dim', no_wrap=True) body.add_column('k', style='dim', no_wrap=True)
body.add_column('v', overflow='fold') body.add_column('v', overflow='fold')
@ -516,9 +526,12 @@ def _cmd_eval_run(args) -> int:
report.save(f'{out_dir}/reports/{name}.report.json') report.save(f'{out_dir}/reports/{name}.report.json')
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f: with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
f.write(render(report, style='text')) f.write(render(report, style='text'))
if len(args.datasets) == 1 or args.verbose: if args.verbose:
if not _print_result_panel(console, report, _time.time() - t0): if not _print_result_panel(console, report, _time.time() - t0):
print(render(report, style=args.style)) print(render(report, style=args.style))
elif len(args.datasets) == 1:
if console is None:
print(render(report, style=args.style))
primary = next(iter(report.metrics), '') primary = next(iter(report.metrics), '')
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0) secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
for s in report.samples) for s in report.samples)
@ -563,7 +576,7 @@ def _cmd_eval_run(args) -> int:
_print_benchmark_result(console, i + 1, total_runs, name, _print_benchmark_result(console, i + 1, total_runs, name,
'failed', _time.time() - t0) 'failed', _time.time() - t0)
if len(rows) > 1: if rows:
_print_summary_table(console, rows) _print_summary_table(console, rows)
if out_dir: if out_dir:
import csv as _csv import csv as _csv
@ -613,25 +626,18 @@ def _cmd_eval_run(args) -> int:
# artifacts notice: tell the user where everything landed (or how to save); # artifacts notice: tell the user where everything landed (or how to save);
# rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...) # rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...)
def _notice(label, *paths): def _notice(label, *paths):
if console is not None:
parts = []
for p in paths:
ap = os.path.abspath(p)
parts.append(f'[link=file://{ap}]{ap}[/link]')
console.print(f'\n[bold]{label}[/bold] -> ' + ' · '.join(parts))
else:
print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths)) print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths))
ok_n = sum(1 for r in rows if r['ok']) ok_n = sum(1 for r in rows if r['ok'])
if out_dir: if out_dir:
ap = os.path.abspath(out_dir) ap = os.path.abspath(out_dir)
if console is not None:
mark = '[green]✓[/green]' if ok_n == len(rows) else '[yellow]◐[/yellow]' mark = '[green]✓[/green]' if ok_n == len(rows) else '[yellow]◐[/yellow]'
if console is not None:
console.print( console.print(
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n' f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 [link=file://{ap}]{ap}[/link]\n' f' 结果 {ap}\n'
f' ├─ [link=file://{ap}/reports]reports/<bench>.report.json[/link]\n' f' ├─ reports/<bench>.report.json\n'
f' └─ [link=file://{ap}/viz/summary.md]viz/summary.md[/link] (+ summary.csv)') f' └─ viz/summary.md (+ summary.csv)')
else: else:
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n' print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md') f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md')