Rich result panel: metrics+bar with score colors, run stats (wall/model-time/throughput/tokens in-out/tok-s), top+bottom group highlights, perf row
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora
parent
b6c473ac26
commit
5c3d622f41
@ -236,6 +236,75 @@ def _compose_model_spec(args):
|
||||
return _model_with_flags(model, args)
|
||||
|
||||
|
||||
|
||||
def _print_result_panel(console, rep, wall_s: float = 0.0):
|
||||
"""Rich result panel: metrics + throughput + health + top/bottom groups."""
|
||||
if console is None:
|
||||
return False
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
|
||||
def color(v):
|
||||
return 'green' if v >= 0.8 else ('yellow' if v >= 0.6 else 'red')
|
||||
|
||||
metrics = [(m, v) for m, v in rep.metrics.items()
|
||||
if isinstance(v, (int, float))]
|
||||
info = rep.metric_groups.get('run_info', {}) or {}
|
||||
model_lat = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
||||
for s in rep.samples)
|
||||
tok_in = info.get('gen_input_tokens', 0) or 0
|
||||
tok_out = info.get('gen_output_tokens', 0) or 0
|
||||
tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0
|
||||
tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0
|
||||
|
||||
title = f'{rep.dataset} · {rep.model or "?"}'
|
||||
body = Table(show_header=False, box=None, padding=(0, 2))
|
||||
body.add_column('k', style='dim', no_wrap=True)
|
||||
body.add_column('v', overflow='fold')
|
||||
for m, v in metrics:
|
||||
if m == 'extraction_failure_rate':
|
||||
continue
|
||||
bar = Text('█' * int(round(v * 24)) + '·' * (24 - int(round(v * 24))))
|
||||
bar.stylize(color(v))
|
||||
body.add_row(m, Text.assemble(f'{v * 100:6.1f}% ', bar))
|
||||
stats = []
|
||||
stats.append(f'n={rep.num_samples}')
|
||||
if wall_s >= 1:
|
||||
stats.append(f'wall={wall_s:.0f}s')
|
||||
if model_lat >= 1:
|
||||
stats.append(f'model-time={model_lat:.0f}s')
|
||||
if tput:
|
||||
stats.append(f'{tput:.1f} samples/s')
|
||||
if tok_in or tok_out:
|
||||
stats.append(f'tokens in={tok_in:,} out={tok_out:,}')
|
||||
if tps:
|
||||
stats.append(f'{tps:.0f} tok/s out')
|
||||
if rep.num_failed_extractions:
|
||||
stats.append(f'[red]extract-fail={rep.num_failed_extractions}[/red]')
|
||||
body.add_row('run', ' '.join(stats))
|
||||
for gname, groups in rep.metric_groups.items():
|
||||
if gname in ('run_info', 'perf') or gname.startswith('agg_error'):
|
||||
continue
|
||||
numeric = {g: v for g, v in groups.items() if isinstance(v, (int, float))}
|
||||
if len(numeric) < 2:
|
||||
continue
|
||||
rank = sorted(numeric.items(), key=lambda kv: -kv[1])
|
||||
show = [f'[green]{g} {v * 100:.0f}%[/green]' for g, v in rank[:3]]
|
||||
show += [f'[red]{g} {v * 100:.0f}%[/red]' for g, v in rank[-2:]]
|
||||
body.add_row(gname, ' '.join(show) + f' (+{len(numeric) - 5} more)'
|
||||
if len(numeric) > 5 else ' '.join(show))
|
||||
perf = rep.metric_groups.get('perf') or {}
|
||||
if perf:
|
||||
keys = ('ttft_mean_s', 'ttft_p90_s', 'latency_mean_s', 'latency_p90_s',
|
||||
'output_tps', 'success_rate', 'retry_rate')
|
||||
pstats = [f'{k}={perf[k]:.2f}' if isinstance(perf.get(k), float) and perf[k] < 10
|
||||
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
|
||||
if pstats:
|
||||
body.add_row('perf', ' '.join(pstats))
|
||||
console.print(Panel(body, title=title, border_style='blue', expand=False))
|
||||
return True
|
||||
|
||||
def _compose_judge_spec(args):
|
||||
"""--judge accepts a bare model name (with --judge-api-url) or a full
|
||||
legacy spec; keep both working like the main model flags."""
|
||||
@ -248,6 +317,75 @@ def _compose_judge_spec(args):
|
||||
return judge or None
|
||||
|
||||
|
||||
|
||||
def _print_result_panel(console, rep, wall_s: float = 0.0):
|
||||
"""Rich result panel: metrics + throughput + health + top/bottom groups."""
|
||||
if console is None:
|
||||
return False
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
|
||||
def color(v):
|
||||
return 'green' if v >= 0.8 else ('yellow' if v >= 0.6 else 'red')
|
||||
|
||||
metrics = [(m, v) for m, v in rep.metrics.items()
|
||||
if isinstance(v, (int, float))]
|
||||
info = rep.metric_groups.get('run_info', {}) or {}
|
||||
model_lat = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
||||
for s in rep.samples)
|
||||
tok_in = info.get('gen_input_tokens', 0) or 0
|
||||
tok_out = info.get('gen_output_tokens', 0) or 0
|
||||
tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0
|
||||
tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0
|
||||
|
||||
title = f'{rep.dataset} · {rep.model or "?"}'
|
||||
body = Table(show_header=False, box=None, padding=(0, 2))
|
||||
body.add_column('k', style='dim', no_wrap=True)
|
||||
body.add_column('v', overflow='fold')
|
||||
for m, v in metrics:
|
||||
if m == 'extraction_failure_rate':
|
||||
continue
|
||||
bar = Text('█' * int(round(v * 24)) + '·' * (24 - int(round(v * 24))))
|
||||
bar.stylize(color(v))
|
||||
body.add_row(m, Text.assemble(f'{v * 100:6.1f}% ', bar))
|
||||
stats = []
|
||||
stats.append(f'n={rep.num_samples}')
|
||||
if wall_s >= 1:
|
||||
stats.append(f'wall={wall_s:.0f}s')
|
||||
if model_lat >= 1:
|
||||
stats.append(f'model-time={model_lat:.0f}s')
|
||||
if tput:
|
||||
stats.append(f'{tput:.1f} samples/s')
|
||||
if tok_in or tok_out:
|
||||
stats.append(f'tokens in={tok_in:,} out={tok_out:,}')
|
||||
if tps:
|
||||
stats.append(f'{tps:.0f} tok/s out')
|
||||
if rep.num_failed_extractions:
|
||||
stats.append(f'[red]extract-fail={rep.num_failed_extractions}[/red]')
|
||||
body.add_row('run', ' '.join(stats))
|
||||
for gname, groups in rep.metric_groups.items():
|
||||
if gname in ('run_info', 'perf') or gname.startswith('agg_error'):
|
||||
continue
|
||||
numeric = {g: v for g, v in groups.items() if isinstance(v, (int, float))}
|
||||
if len(numeric) < 2:
|
||||
continue
|
||||
rank = sorted(numeric.items(), key=lambda kv: -kv[1])
|
||||
show = [f'[green]{g} {v * 100:.0f}%[/green]' for g, v in rank[:3]]
|
||||
show += [f'[red]{g} {v * 100:.0f}%[/red]' for g, v in rank[-2:]]
|
||||
body.add_row(gname, ' '.join(show) + f' (+{len(numeric) - 5} more)'
|
||||
if len(numeric) > 5 else ' '.join(show))
|
||||
perf = rep.metric_groups.get('perf') or {}
|
||||
if perf:
|
||||
keys = ('ttft_mean_s', 'ttft_p90_s', 'latency_mean_s', 'latency_p90_s',
|
||||
'output_tps', 'success_rate', 'retry_rate')
|
||||
pstats = [f'{k}={perf[k]:.2f}' if isinstance(perf.get(k), float) and perf[k] < 10
|
||||
else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None]
|
||||
if pstats:
|
||||
body.add_row('perf', ' '.join(pstats))
|
||||
console.print(Panel(body, title=title, border_style='blue', expand=False))
|
||||
return True
|
||||
|
||||
def _compose_judge_spec(args):
|
||||
"""--judge accepts a bare model name (with --judge-api-url) or a full
|
||||
legacy spec; both keep working, mirroring the main model flags."""
|
||||
@ -343,7 +481,8 @@ def _cmd_eval_run(args) -> int:
|
||||
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(render(report, style='text'))
|
||||
if len(args.datasets) == 1 or args.verbose:
|
||||
print(render(report, style=args.style))
|
||||
if not _print_result_panel(console, report, _time.time() - t0):
|
||||
print(render(report, style=args.style))
|
||||
primary = next(iter(report.metrics), '')
|
||||
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
||||
for s in report.samples)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user