diff --git a/evalharness/cli.py b/evalharness/cli.py index 411c08e..d4c5613 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -510,9 +510,10 @@ def _cmd_eval_run(args) -> int: 'secs': round(secs_total, 1), 'wall': round(_time.time() - t0, 1), 'hours': round(secs_total / 3600, 2), + 'tok_in': info.get('gen_input_tokens', 0) or 0, + 'tok_out': info.get('gen_output_tokens', 0) or 0, 'tokens': (info.get('gen_input_tokens', 0) or 0) + (info.get('gen_output_tokens', 0) or 0), - 'tok_out': info.get('gen_output_tokens', 0) or 0, 'groups': groups, 'ok': True}) _print_benchmark_result(console, i + 1, total_runs, name, 'done', _time.time() - t0) @@ -574,17 +575,30 @@ def _cmd_eval_run(args) -> int: f"{r.get('n', '')} | {t} | {st} |\n") if out_dir: print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)') - # artifacts notice: tell the user where everything landed (or how to save) + # artifacts notice: tell the user where everything landed (or how to save); + # rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...) + def _notice(label, *paths): + if console is not None: + parts = [] + for p in paths: + ap = os.path.abspath(p) + parts.append(f'[link=file://{ap}]{ap}[/link]') + console.print(f'\n[bold]{label}[/bold] -> ' + ' · '.join(parts)) + else: + print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths)) + if len(rows) > 1: if out_dir: - print(f'\n结果已生成 -> {out_dir}/reports/.report.json · ' - f'{out_dir}/viz/summary.md') + _notice('结果已生成', f'{out_dir}/reports/.report.json', + f'{out_dir}/viz/summary.md') else: print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)') elif rows and rows[0]['ok']: saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '') - print(f'\n结果已生成 -> {saved}' if saved else - '\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告') + if saved: + _notice('结果已生成', saved) + else: + print('\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告') return 0 if all(r['ok'] for r in rows) else 1 @@ -606,31 +620,33 @@ def _print_summary_table(console, rows): title_style='bold', expand=False) for col, just in (('benchmark', 'left'), ('metric', 'left'), ('score', 'right'), ('n', 'right'), ('time', 'right'), - ('tokens', 'right'), ('tok/s', 'right'), ('note', 'left')): + ('tok in', 'right'), ('tok out', 'right'), + ('in/s', 'right'), ('out/s', 'right')): t.add_column(col, justify=just) for r in rows: v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' wall = r.get('wall') or r.get('secs') or 0 tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s' - tok = f"{r.get('tokens', 0):,}" if r.get('tokens') else '—' - tps = (r.get('tok_out', 0) / wall) if (wall > 1 and r.get('tok_out')) else 0 - tp = f'{tps:.0f}' if tps else '—' - if not r['ok']: - note = f"[red]{str(r.get('err', ''))[:34]}[/red]" - elif r.get('extract_fail'): - note = f"[yellow]extract-fail {r['extract_fail']}[/yellow]" - else: - note = '' - t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, tok, tp, note, + ti, to = r.get('tok_in', 0), r.get('tok_out', 0) + tin = f'{ti:,}' if ti else '—' + tout = f'{to:,}' if to else '—' + tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—' + tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—' + t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, + tin, tout, tis, tos, style='green' if r['ok'] else 'red') wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) - tok_all = sum(r.get('tokens', 0) for r in rows) + ti_all = sum(r.get('tok_in', 0) for r in rows) + to_all = sum(r.get('tok_out', 0) for r in rows) n_all = sum(r.get('n', 0) or 0 for r in rows if isinstance(r.get('n'), int)) tm_all = f'{wall_all / 3600:.2f}h' if wall_all >= 3600 else f'{wall_all:.0f}s' t.add_section() + tis_all = f'{ti_all / wall_all:.0f}' if wall_all > 1 else '' + tos_all = f'{to_all / wall_all:.0f}' if wall_all > 1 else '' t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', - str(n_all), tm_all, f'{tok_all:,}', '', '') + str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}', + tis_all, tos_all) console.print(t) return print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')