Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run Plan Samples row: show the actual sampling config instead of the confusing placeholder

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:23:49 +00:00
parent e9c0b5be77
commit 1ef230f4c1
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
evalharness/cli.py
View File

@ -158,17 +158,24 @@ def _print_run_plan(console, args, model_spec):
provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—' provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—'
api_url = getattr(args, 'api_url', '') or '—' api_url = getattr(args, 'api_url', '') or '—'
model_name = getattr(args, 'model', '') or 'predictions file' model_name = getattr(args, 'model', '') or 'predictions file'
# sampling summary: what the run will actually evaluate
if getattr(args, 'limit', None):
samples = f'up to {args.limit} total (--limit)'
elif getattr(args, 'limit_per_task', None):
samples = f'up to {args.limit_per_task} per subject (--limit-per-task)'
else:
samples = 'full dataset (counted when each loads)'
if console is None: if console is None:
print(f'=== {title} ===') print(f'=== {title} ===')
print(f'Provider: {provider}') print(f'Provider: {provider}')
print(f'API URL: {api_url}') print(f'API URL: {api_url}')
print(f'Model: {model_name}') print(f'Model: {model_name}')
print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}') print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}')
print(f'Samples: {samples}')
print(f'Concurrency: {args.concurrency} | Thinking: ' print(f'Concurrency: {args.concurrency} | Thinking: '
f'{"enabled" if not args.disable_thinking else "disabled"} | ' f'{"enabled" if not args.disable_thinking else "disabled"} | '
f'Performance: {"on" if args.perf else "off"}') f'Performance: {"on" if args.perf else "off"}')
print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}') print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}')
print('Samples: counted after each dataset is loaded')
return return
from rich.panel import Panel from rich.panel import Panel
@ -181,7 +188,7 @@ def _print_run_plan(console, args, model_spec):
table.add_row('API URL', api_url) table.add_row('API URL', api_url)
table.add_row('Model', model_name) table.add_row('Model', model_name)
table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}') table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}')
table.add_row('Samples', 'counted while loading each benchmark') table.add_row('Samples', samples)
table.add_row('Concurrency', str(args.concurrency)) table.add_row('Concurrency', str(args.concurrency))
table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]') table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]')
table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]') table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')