From 1ef230f4c1e746075e1fbde96e56d2ade872bdb7 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 10 Sep 2026 10:23:49 +0000 Subject: [PATCH] Run Plan Samples row: show the actual sampling config instead of the confusing placeholder Co-Authored-By: Claude --- evalharness/cli.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 4a13b19..19a7a2e 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -158,17 +158,24 @@ def _print_run_plan(console, args, model_spec): provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—' api_url = getattr(args, 'api_url', '') or '—' model_name = getattr(args, 'model', '') or 'predictions file' + # sampling summary: what the run will actually evaluate + if getattr(args, 'limit', None): + samples = f'up to {args.limit} total (--limit)' + elif getattr(args, 'limit_per_task', None): + samples = f'up to {args.limit_per_task} per subject (--limit-per-task)' + else: + samples = 'full dataset (counted when each loads)' if console is None: print(f'=== {title} ===') print(f'Provider: {provider}') print(f'API URL: {api_url}') print(f'Model: {model_name}') print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}') + print(f'Samples: {samples}') print(f'Concurrency: {args.concurrency} | Thinking: ' f'{"enabled" if not args.disable_thinking else "disabled"} | ' f'Performance: {"on" if args.perf else "off"}') print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}') - print('Samples: counted after each dataset is loaded') return from rich.panel import Panel @@ -181,7 +188,7 @@ def _print_run_plan(console, args, model_spec): table.add_row('API URL', api_url) table.add_row('Model', model_name) table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}') - table.add_row('Samples', 'counted while loading each benchmark') + table.add_row('Samples', samples) table.add_row('Concurrency', str(args.concurrency)) table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]') table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')