From 82d9d03b88317e33c3248a9864c8dc05e095e0d2 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 10 Sep 2026 09:49:36 +0000 Subject: [PATCH] Run Summary: tokens/throughput/note columns + totals row; final artifacts notice (where results were saved, or how to save) Co-Authored-By: Claude --- evalharness/cli.py | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 448dee0..411c08e 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -503,11 +503,16 @@ def _cmd_eval_run(args) -> int: groups = {k: v for k, v in report.metric_groups.items() if isinstance(v, dict) and k not in ('run_info',) and not k.startswith('agg_error')} + info = report.metric_groups.get('run_info', {}) or {} rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), 'n': report.num_samples, 'extract_fail': report.num_failed_extractions, 'secs': round(secs_total, 1), + 'wall': round(_time.time() - t0, 1), 'hours': round(secs_total / 3600, 2), + 'tokens': (info.get('gen_input_tokens', 0) or 0) + + (info.get('gen_output_tokens', 0) or 0), + 'tok_out': info.get('gen_output_tokens', 0) or 0, 'groups': groups, 'ok': True}) _print_benchmark_result(console, i + 1, total_runs, name, 'done', _time.time() - t0) @@ -569,6 +574,17 @@ def _cmd_eval_run(args) -> int: f"{r.get('n', '')} | {t} | {st} |\n") if out_dir: print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)') + # artifacts notice: tell the user where everything landed (or how to save) + if len(rows) > 1: + if out_dir: + print(f'\n结果已生成 -> {out_dir}/reports/.report.json · ' + f'{out_dir}/viz/summary.md') + else: + print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)') + elif rows and rows[0]['ok']: + saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '') + print(f'\n结果已生成 -> {saved}' if saved else + '\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告') return 0 if all(r['ok'] for r in rows) else 1 @@ -589,15 +605,32 @@ def _print_summary_table(console, rows): t = Table(title='Run Summary', header_style='bold cyan', title_style='bold', expand=False) for col, just in (('benchmark', 'left'), ('metric', 'left'), - ('score', 'right'), ('n', 'right'), ('time', 'right')): + ('score', 'right'), ('n', 'right'), ('time', 'right'), + ('tokens', 'right'), ('tok/s', 'right'), ('note', 'left')): t.add_column(col, justify=just) for r in rows: v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' - h = r.get('hours') or 0 - tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" - style = 'green' if r['ok'] else 'red' - t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, - style=style) + wall = r.get('wall') or r.get('secs') or 0 + tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s' + tok = f"{r.get('tokens', 0):,}" if r.get('tokens') else '—' + tps = (r.get('tok_out', 0) / wall) if (wall > 1 and r.get('tok_out')) else 0 + tp = f'{tps:.0f}' if tps else '—' + if not r['ok']: + note = f"[red]{str(r.get('err', ''))[:34]}[/red]" + elif r.get('extract_fail'): + note = f"[yellow]extract-fail {r['extract_fail']}[/yellow]" + else: + note = '' + t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, tok, tp, note, + style='green' if r['ok'] else 'red') + wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) + tok_all = sum(r.get('tokens', 0) for r in rows) + n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int)) + tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s' + t.add_section() + t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', + f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', + str(n_all), tm_all, f'{tok_all:,}', '', '') console.print(t) return print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')