Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Center run plan, result panel, and Run Summary table; add lat p50/p90 + truncation columns

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:03:19 +00:00
parent b4d39c560e
commit c1896e0a63
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
evalharness/cli.py
View File

@ -187,7 +187,8 @@ def _print_run_plan(console, args, model_spec):
table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]') table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')
table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]') table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]')
table.add_row('Output', args.out_dir or '[dim](not specified)[/dim]') table.add_row('Output', args.out_dir or '[dim](not specified)[/dim]')
console.print(Panel(table, title=title, border_style='blue', expand=False)) console.print(Panel(table, title=title, border_style='blue', expand=False),
justify='center')
def _print_phase(console, index, total, name, message): def _print_phase(console, index, total, name, message):
@ -309,7 +310,8 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None] else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
if pstats: if pstats:
body.add_row('perf', ' '.join(pstats)) body.add_row('perf', ' '.join(pstats))
console.print(Panel(body, title=title, border_style='blue', expand=False)) console.print(Panel(body, title=title, border_style='blue', expand=False),
justify='center')
return True return True
def _compose_judge_spec(args): def _compose_judge_spec(args):
@ -397,7 +399,8 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None] else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
if pstats: if pstats:
body.add_row('perf', ' '.join(pstats)) body.add_row('perf', ' '.join(pstats))
console.print(Panel(body, title=title, border_style='blue', expand=False)) console.print(Panel(body, title=title, border_style='blue', expand=False),
justify='center')
return True return True
def _compose_judge_spec(args): def _compose_judge_spec(args):
@ -504,6 +507,13 @@ def _cmd_eval_run(args) -> int:
if isinstance(v, dict) and k not in ('run_info',) if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')} and not k.startswith('agg_error')}
info = report.metric_groups.get('run_info', {}) or {} info = report.metric_groups.get('run_info', {}) or {}
lats = sorted(float((s.usage or {}).get('latency_s', 0) or 0)
for s in report.samples
if float((s.usage or {}).get('latency_s', 0) or 0) > 0)
def _pct(q):
return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0
fins = [(s.usage or {}).get('finish_reason', '')
for s in report.samples]
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
'n': report.num_samples, 'n': report.num_samples,
'extract_fail': report.num_failed_extractions, 'extract_fail': report.num_failed_extractions,
@ -514,6 +524,8 @@ def _cmd_eval_run(args) -> int:
'tok_out': info.get('gen_output_tokens', 0) or 0, 'tok_out': info.get('gen_output_tokens', 0) or 0,
'tokens': (info.get('gen_input_tokens', 0) or 0) 'tokens': (info.get('gen_input_tokens', 0) or 0)
+ (info.get('gen_output_tokens', 0) or 0), + (info.get('gen_output_tokens', 0) or 0),
'lat_p50': _pct(0.50), 'lat_p90': _pct(0.90),
'trunc': sum(1 for f in fins if f == 'length'),
'groups': groups, 'ok': True}) 'groups': groups, 'ok': True})
_print_benchmark_result(console, i + 1, total_runs, name, _print_benchmark_result(console, i + 1, total_runs, name,
'done', _time.time() - t0) 'done', _time.time() - t0)
@ -621,7 +633,8 @@ def _print_summary_table(console, rows):
for col, just in (('benchmark', 'left'), ('metric', 'left'), for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'), ('score', 'right'), ('n', 'right'), ('time', 'right'),
('tok in', 'right'), ('tok out', 'right'), ('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right')): ('in/s', 'right'), ('out/s', 'right'),
('lat p50/p90', 'right'), ('trunc', 'right')):
t.add_column(col, justify=just) t.add_column(col, justify=just)
for r in rows: for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
@ -632,8 +645,11 @@ def _print_summary_table(console, rows):
tout = f'{to:,}' if to else '—' tout = f'{to:,}' if to else '—'
tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—' tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—' tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
lat = f"{r.get('lat_p50', 0):.1f}/{r.get('lat_p90', 0):.1f}s" \
if r.get('lat_p50') else '—'
trunc = str(r['trunc']) if r.get('trunc') else '—'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
tin, tout, tis, tos, tin, tout, tis, tos, lat, trunc,
style='green' if r['ok'] else 'red') style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
ti_all = sum(r.get('tok_in', 0) for r in rows) ti_all = sum(r.get('tok_in', 0) for r in rows)
@ -646,8 +662,8 @@ def _print_summary_table(console, rows):
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}', str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all) tis_all, tos_all, '', str(sum(r.get('trunc', 0) for r in rows)))
console.print(t) console.print(t, justify='center')
return return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}') print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')
print('-' * 64) print('-' * 64)