diff --git a/evalharness/cli.py b/evalharness/cli.py index 25991cf..346e7eb 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -163,19 +163,90 @@ def _rich_console(): return None +def _load_bench_cfg(args, name: str) -> dict: + """Merged YAML config for one bench: {default 段, bench 段}. + + Single source for BOTH the run loop and the run-plan sample count + (repeats must agree between the two or the plan under-reports). + """ + from pathlib import Path + + cfg_dir = Path(__file__).parent / 'config' + cfg_name = getattr(args, 'config', '') + if not cfg_name and cfg_dir.exists(): + yamls = sorted(cfg_dir.glob('*.yaml')) + if len(yamls) == 1: + cfg_name = yamls[0].stem # auto: the only config + if not cfg_name: + return {} + cfg_path = cfg_dir / f'{cfg_name}.yaml' + if not cfg_path.exists(): + return {} + try: + import yaml as _yaml + + all_cfg = _yaml.safe_load(open(cfg_path)) or {} + return {**(all_cfg.get('default') or {}), **(all_cfg.get(name) or {})} + except Exception: + return {} + + +def _plan_sample_counts(args): + """(total_samples, total_generations, n_uncached) across the planned + benches, counted from LOCAL cache entries only -- never touches the + network, so the run plan stays instant on cold machines. Uncached + benches simply don't contribute yet. + """ + from evalharness.data import get_dataset + + total = gens = uncached = 0 + for name in getattr(args, 'datasets', []) or []: + try: + ds = get_dataset(name, **_overrides(args)) + cache_file = ds.cache_dir / 'samples.jsonl' + if not cache_file.exists(): + uncached += 1 + continue + with open(cache_file, 'rb') as f: + n = sum(1 for _ in f) + except Exception: + uncached += 1 + continue + if getattr(args, 'limit', None): + n = min(n, args.limit) + total += n + gens += n * int(_load_bench_cfg(args, name).get('repeats', 1) or 1) + return total, gens, uncached + + def _print_run_plan(console, args, model_spec): """Print the important run facts before any dataset work starts.""" title = 'EvalHarness · Run Plan' provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—' api_url = getattr(args, 'api_url', '') or '—' model_name = getattr(args, 'model', '') or 'predictions file' - # sampling summary: what the run will actually evaluate + # sampling summary: prefer REAL cached counts; --limit caps each bench + n_samples, n_gens, n_uncached = _plan_sample_counts(args) + cap = '' if getattr(args, 'limit', None): - samples = f'up to {args.limit} total (--limit)' + cap = f' · ≤{args.limit} per bench (--limit)' elif getattr(args, 'limit_per_task', None): - samples = f'up to {args.limit_per_task} per subject (--limit-per-task)' - else: - samples = 'full dataset (counted when each loads)' + cap = f' · ≤{args.limit_per_task} per subject (--limit-per-task)' + if n_samples: + samples = f'{n_samples:,} samples (cached){cap}' + if n_gens > n_samples: # repeats multiply the real work + samples = (f'{n_samples:,} samples (cached){cap} → ' + f'{n_gens:,} generations (repeats)') + if n_uncached: + samples += f' · {n_uncached} bench(es) not cached yet' + elif n_uncached: + if getattr(args, 'limit', None): + samples = f'up to {args.limit} per bench (--limit), counts when datasets load' + elif getattr(args, 'limit_per_task', None): + samples = (f'up to {args.limit_per_task} per subject ' + '(--limit-per-task), counts when datasets load') + else: + samples = 'full dataset, counts when datasets load (none cached yet)' if console is None: print(f'=== {title} ===') print(f'Provider: {provider}') @@ -584,28 +655,11 @@ def _cmd_eval_run(args) -> int: _emit(f'Dataset ready: {sample_count} samples from {origin}') # YAML config: per-bench generation params, AUTO-LOADED # (single .yaml in config/ = the default; --config overrides) - bench_cfg = {} - import yaml as _yaml - from pathlib import Path as _P - - _cfg_dir = _P(__file__).parent / 'config' - if not _cfg_dir.exists(): - _cfg_dir = _P('/data1/sora/evalharness/EvalHarness/evalharness/config') - _cfg_name = getattr(args, 'config', '') - if not _cfg_name: - _yamls = sorted(_cfg_dir.glob('*.yaml')) if _cfg_dir.exists() else [] - if len(_yamls) == 1: - _cfg_name = _yamls[0].stem # auto: the only config - if _cfg_name: - cfg_path = _cfg_dir / f'{_cfg_name}.yaml' - if cfg_path.exists(): - _all = _yaml.safe_load(open(cfg_path)) or {} - _default = _all.get('default', {}) - bench_cfg = {**_default, **(_all.get(name) or {})} - # strip non-generation keys (they go to run_eval kwargs) - for k in ('judge', 'judge_url', 'env', 'max_turns', - 'limit', 'limit_per_task', 'concurrency'): - bench_cfg.pop(k, None) + bench_cfg = _load_bench_cfg(args, name) + # strip non-generation keys (they go to run_eval kwargs) + for k in ('judge', 'judge_url', 'env', 'max_turns', + 'limit', 'limit_per_task', 'concurrency'): + bench_cfg.pop(k, None) # repeats: run the benchmark N times, report mean ± spread _repeats = int(bench_cfg.pop('repeats', 1) or 1)