Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Center run plan, result panel, and Run Summary table; add lat p50/p90 + truncation columns

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:03:19 +00:00
parent b4d39c560e
commit c1896e0a63
1 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
evalharness/cli.py
View File

@ -187,7 +187,8 @@ def _print_run_plan(console, args, model_spec):
table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')
table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]')
table.add_row('Output', args.out_dir or '[dim](not specified)[/dim]')
console.print(Panel(table, title=title, border_style='blue', expand=False))
console.print(Panel(table, title=title, border_style='blue', expand=False),
justify='center')
def _print_phase(console, index, total, name, message):
@ -309,7 +310,8 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
if pstats:
body.add_row('perf', ' '.join(pstats))
console.print(Panel(body, title=title, border_style='blue', expand=False))
console.print(Panel(body, title=title, border_style='blue', expand=False),
justify='center')
return True
def _compose_judge_spec(args):
@ -397,7 +399,8 @@ def _print_result_panel(console, rep, wall_s: float = 0.0):
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
if pstats:
body.add_row('perf', ' '.join(pstats))
console.print(Panel(body, title=title, border_style='blue', expand=False))
console.print(Panel(body, title=title, border_style='blue', expand=False),
justify='center')
return True
def _compose_judge_spec(args):
@ -504,6 +507,13 @@ def _cmd_eval_run(args) -> int:
if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')}
info = report.metric_groups.get('run_info', {}) or {}
lats = sorted(float((s.usage or {}).get('latency_s', 0) or 0)
for s in report.samples
if float((s.usage or {}).get('latency_s', 0) or 0) > 0)
def _pct(q):
return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0
fins = [(s.usage or {}).get('finish_reason', '')
for s in report.samples]
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
'n': report.num_samples,
'extract_fail': report.num_failed_extractions,
@ -514,6 +524,8 @@ def _cmd_eval_run(args) -> int:
'tok_out': info.get('gen_output_tokens', 0) or 0,
'tokens': (info.get('gen_input_tokens', 0) or 0)
+ (info.get('gen_output_tokens', 0) or 0),
'lat_p50': _pct(0.50), 'lat_p90': _pct(0.90),
'trunc': sum(1 for f in fins if f == 'length'),
'groups': groups, 'ok': True})
_print_benchmark_result(console, i + 1, total_runs, name,
'done', _time.time() - t0)
@ -621,7 +633,8 @@ def _print_summary_table(console, rows):
for col, just in (('benchmark', 'left'), ('metric', 'left'),
('score', 'right'), ('n', 'right'), ('time', 'right'),
('tok in', 'right'), ('tok out', 'right'),
('in/s', 'right'), ('out/s', 'right')):
('in/s', 'right'), ('out/s', 'right'),
('lat p50/p90', 'right'), ('trunc', 'right')):
t.add_column(col, justify=just)
for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
@ -632,8 +645,11 @@ def _print_summary_table(console, rows):
tout = f'{to:,}' if to else '—'
tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—'
tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—'
lat = f"{r.get('lat_p50', 0):.1f}/{r.get('lat_p90', 0):.1f}s" \
if r.get('lat_p50') else '—'
trunc = str(r['trunc']) if r.get('trunc') else '—'
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
tin, tout, tis, tos,
tin, tout, tis, tos, lat, trunc,
style='green' if r['ok'] else 'red')
wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows)
ti_all = sum(r.get('tok_in', 0) for r in rows)
@ -646,8 +662,8 @@ def _print_summary_table(console, rows):
t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '',
f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok',
str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}',
tis_all, tos_all)
console.print(t)
tis_all, tos_all, '', str(sum(r.get('trunc', 0) for r in rows)))
console.print(t, justify='center')
return
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')
print('-' * 64)