From 5c3d622f4106ff6c2f9ebf44a21c3d5b821df0cd Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 10 Sep 2026 09:20:44 +0000 Subject: [PATCH] Rich result panel: metrics+bar with score colors, run stats (wall/model-time/throughput/tokens in-out/tok-s), top+bottom group highlights, perf row Co-Authored-By: Claude --- evalharness/cli.py | 141 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index a0f50ba..6874585 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -236,6 +236,75 @@ def _compose_model_spec(args): return _model_with_flags(model, args) + +def _print_result_panel(console, rep, wall_s: float = 0.0): + """Rich result panel: metrics + throughput + health + top/bottom groups.""" + if console is None: + return False + from rich.panel import Panel + from rich.table import Table + from rich.text import Text + + def color(v): + return 'green' if v >= 0.8 else ('yellow' if v >= 0.6 else 'red') + + metrics = [(m, v) for m, v in rep.metrics.items() + if isinstance(v, (int, float))] + info = rep.metric_groups.get('run_info', {}) or {} + model_lat = sum(float((s.usage or {}).get('latency_s', 0) or 0) + for s in rep.samples) + tok_in = info.get('gen_input_tokens', 0) or 0 + tok_out = info.get('gen_output_tokens', 0) or 0 + tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0 + tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0 + + title = f'{rep.dataset} · {rep.model or "?"}' + body = Table(show_header=False, box=None, padding=(0, 2)) + body.add_column('k', style='dim', no_wrap=True) + body.add_column('v', overflow='fold') + for m, v in metrics: + if m == 'extraction_failure_rate': + continue + bar = Text('█' * int(round(v * 24)) + '·' * (24 - int(round(v * 24)))) + bar.stylize(color(v)) + body.add_row(m, Text.assemble(f'{v * 100:6.1f}% ', bar)) + stats = [] + stats.append(f'n={rep.num_samples}') + if wall_s >= 1: + stats.append(f'wall={wall_s:.0f}s') + if model_lat >= 1: + stats.append(f'model-time={model_lat:.0f}s') + if tput: + stats.append(f'{tput:.1f} samples/s') + if tok_in or tok_out: + stats.append(f'tokens in={tok_in:,} out={tok_out:,}') + if tps: + stats.append(f'{tps:.0f} tok/s out') + if rep.num_failed_extractions: + stats.append(f'[red]extract-fail={rep.num_failed_extractions}[/red]') + body.add_row('run', ' '.join(stats)) + for gname, groups in rep.metric_groups.items(): + if gname in ('run_info', 'perf') or gname.startswith('agg_error'): + continue + numeric = {g: v for g, v in groups.items() if isinstance(v, (int, float))} + if len(numeric) < 2: + continue + rank = sorted(numeric.items(), key=lambda kv: -kv[1]) + show = [f'[green]{g} {v * 100:.0f}%[/green]' for g, v in rank[:3]] + show += [f'[red]{g} {v * 100:.0f}%[/red]' for g, v in rank[-2:]] + body.add_row(gname, ' '.join(show) + f' (+{len(numeric) - 5} more)' + if len(numeric) > 5 else ' '.join(show)) + perf = rep.metric_groups.get('perf') or {} + if perf: + keys = ('ttft_mean_s', 'ttft_p90_s', 'latency_mean_s', 'latency_p90_s', + 'output_tps', 'success_rate', 'retry_rate') + pstats = [f'{k}={perf[k]:.2f}' if isinstance(perf.get(k), float) and perf[k] < 10 + else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None] + if pstats: + body.add_row('perf', ' '.join(pstats)) + console.print(Panel(body, title=title, border_style='blue', expand=False)) + return True + def _compose_judge_spec(args): """--judge accepts a bare model name (with --judge-api-url) or a full legacy spec; keep both working like the main model flags.""" @@ -248,6 +317,75 @@ def _compose_judge_spec(args): return judge or None + +def _print_result_panel(console, rep, wall_s: float = 0.0): + """Rich result panel: metrics + throughput + health + top/bottom groups.""" + if console is None: + return False + from rich.panel import Panel + from rich.table import Table + from rich.text import Text + + def color(v): + return 'green' if v >= 0.8 else ('yellow' if v >= 0.6 else 'red') + + metrics = [(m, v) for m, v in rep.metrics.items() + if isinstance(v, (int, float))] + info = rep.metric_groups.get('run_info', {}) or {} + model_lat = sum(float((s.usage or {}).get('latency_s', 0) or 0) + for s in rep.samples) + tok_in = info.get('gen_input_tokens', 0) or 0 + tok_out = info.get('gen_output_tokens', 0) or 0 + tput = rep.num_samples / wall_s if wall_s > 0.5 else 0.0 + tps = tok_out / wall_s if (wall_s > 0.5 and tok_out) else 0.0 + + title = f'{rep.dataset} · {rep.model or "?"}' + body = Table(show_header=False, box=None, padding=(0, 2)) + body.add_column('k', style='dim', no_wrap=True) + body.add_column('v', overflow='fold') + for m, v in metrics: + if m == 'extraction_failure_rate': + continue + bar = Text('█' * int(round(v * 24)) + '·' * (24 - int(round(v * 24)))) + bar.stylize(color(v)) + body.add_row(m, Text.assemble(f'{v * 100:6.1f}% ', bar)) + stats = [] + stats.append(f'n={rep.num_samples}') + if wall_s >= 1: + stats.append(f'wall={wall_s:.0f}s') + if model_lat >= 1: + stats.append(f'model-time={model_lat:.0f}s') + if tput: + stats.append(f'{tput:.1f} samples/s') + if tok_in or tok_out: + stats.append(f'tokens in={tok_in:,} out={tok_out:,}') + if tps: + stats.append(f'{tps:.0f} tok/s out') + if rep.num_failed_extractions: + stats.append(f'[red]extract-fail={rep.num_failed_extractions}[/red]') + body.add_row('run', ' '.join(stats)) + for gname, groups in rep.metric_groups.items(): + if gname in ('run_info', 'perf') or gname.startswith('agg_error'): + continue + numeric = {g: v for g, v in groups.items() if isinstance(v, (int, float))} + if len(numeric) < 2: + continue + rank = sorted(numeric.items(), key=lambda kv: -kv[1]) + show = [f'[green]{g} {v * 100:.0f}%[/green]' for g, v in rank[:3]] + show += [f'[red]{g} {v * 100:.0f}%[/red]' for g, v in rank[-2:]] + body.add_row(gname, ' '.join(show) + f' (+{len(numeric) - 5} more)' + if len(numeric) > 5 else ' '.join(show)) + perf = rep.metric_groups.get('perf') or {} + if perf: + keys = ('ttft_mean_s', 'ttft_p90_s', 'latency_mean_s', 'latency_p90_s', + 'output_tps', 'success_rate', 'retry_rate') + pstats = [f'{k}={perf[k]:.2f}' if isinstance(perf.get(k), float) and perf[k] < 10 + else f'{k}={perf[k]}' for k in keys if perf.get(k) is not None] + if pstats: + body.add_row('perf', ' '.join(pstats)) + console.print(Panel(body, title=title, border_style='blue', expand=False)) + return True + def _compose_judge_spec(args): """--judge accepts a bare model name (with --judge-api-url) or a full legacy spec; both keep working, mirroring the main model flags.""" @@ -343,7 +481,8 @@ def _cmd_eval_run(args) -> int: with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f: f.write(render(report, style='text')) if len(args.datasets) == 1 or args.verbose: - print(render(report, style=args.style)) + if not _print_result_panel(console, report, _time.time() - t0): + print(render(report, style=args.style)) primary = next(iter(report.metrics), '') secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0) for s in report.samples)