Run plan: show real sample/generation counts from local cache

Samples row said 'full dataset (counted when each loads)' -- now it
counts cached samples.jsonl entries (never touches the network, plan
stays instant on cold machines) and multiplies by YAML repeats:

  12,957 samples (cached) → 13,617 generations (repeats)

--limit caps each bench before summing; uncached benches are flagged.
YAML config loading extracted to _load_bench_cfg, shared by the run
loop and the plan so repeats can't disagree between the two.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -163,19 +163,90 @@ def _rich_console():
         return None
 
 
+def _load_bench_cfg(args, name: str) -> dict:
+    """Merged YAML config for one bench: {default 段, bench 段}.
+
+    Single source for BOTH the run loop and the run-plan sample count
+    (repeats must agree between the two or the plan under-reports).
+    """
+    from pathlib import Path
+
+    cfg_dir = Path(__file__).parent / 'config'
+    cfg_name = getattr(args, 'config', '')
+    if not cfg_name and cfg_dir.exists():
+        yamls = sorted(cfg_dir.glob('*.yaml'))
+        if len(yamls) == 1:
+            cfg_name = yamls[0].stem  # auto: the only config
+    if not cfg_name:
+        return {}
+    cfg_path = cfg_dir / f'{cfg_name}.yaml'
+    if not cfg_path.exists():
+        return {}
+    try:
+        import yaml as _yaml
+
+        all_cfg = _yaml.safe_load(open(cfg_path)) or {}
+        return {**(all_cfg.get('default') or {}), **(all_cfg.get(name) or {})}
+    except Exception:
+        return {}
+
+
+def _plan_sample_counts(args):
+    """(total_samples, total_generations, n_uncached) across the planned
+    benches, counted from LOCAL cache entries only -- never touches the
+    network, so the run plan stays instant on cold machines. Uncached
+    benches simply don't contribute yet.
+    """
+    from evalharness.data import get_dataset
+
+    total = gens = uncached = 0
+    for name in getattr(args, 'datasets', []) or []:
+        try:
+            ds = get_dataset(name, **_overrides(args))
+            cache_file = ds.cache_dir / 'samples.jsonl'
+            if not cache_file.exists():
+                uncached += 1
+                continue
+            with open(cache_file, 'rb') as f:
+                n = sum(1 for _ in f)
+        except Exception:
+            uncached += 1
+            continue
+        if getattr(args, 'limit', None):
+            n = min(n, args.limit)
+        total += n
+        gens += n * int(_load_bench_cfg(args, name).get('repeats', 1) or 1)
+    return total, gens, uncached
+
+
 def _print_run_plan(console, args, model_spec):
     """Print the important run facts before any dataset work starts."""
     title = 'EvalHarness · Run Plan'
     provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—'
     api_url = getattr(args, 'api_url', '') or '—'
     model_name = getattr(args, 'model', '') or 'predictions file'
-    # sampling summary: what the run will actually evaluate
+    # sampling summary: prefer REAL cached counts; --limit caps each bench
+    n_samples, n_gens, n_uncached = _plan_sample_counts(args)
+    cap = ''
     if getattr(args, 'limit', None):
-        samples = f'up to {args.limit} total (--limit)'
+        cap = f' · ≤{args.limit} per bench (--limit)'
     elif getattr(args, 'limit_per_task', None):
-        samples = f'up to {args.limit_per_task} per subject (--limit-per-task)'
+        cap = f' · ≤{args.limit_per_task} per subject (--limit-per-task)'
-    else:
+    if n_samples:
-        samples = 'full dataset (counted when each loads)'
+        samples = f'{n_samples:,} samples (cached){cap}'
+        if n_gens > n_samples:  # repeats multiply the real work
+            samples = (f'{n_samples:,} samples (cached){cap} → '
+                       f'{n_gens:,} generations (repeats)')
+        if n_uncached:
+            samples += f' · {n_uncached} bench(es) not cached yet'
+    elif n_uncached:
+        if getattr(args, 'limit', None):
+            samples = f'up to {args.limit} per bench (--limit), counts when datasets load'
+        elif getattr(args, 'limit_per_task', None):
+            samples = (f'up to {args.limit_per_task} per subject '
+                       '(--limit-per-task), counts when datasets load')
+        else:
+            samples = 'full dataset, counts when datasets load (none cached yet)'
     if console is None:
         print(f'=== {title} ===')
         print(f'Provider: {provider}')
@@ -584,28 +655,11 @@ def _cmd_eval_run(args) -> int:
             _emit(f'Dataset ready: {sample_count} samples from {origin}')
             # YAML config: per-bench generation params, AUTO-LOADED
             # (single .yaml in config/ = the default; --config overrides)
-            bench_cfg = {}
+            bench_cfg = _load_bench_cfg(args, name)
-            import yaml as _yaml
+            # strip non-generation keys (they go to run_eval kwargs)
-            from pathlib import Path as _P
+            for k in ('judge', 'judge_url', 'env', 'max_turns',
-
+                      'limit', 'limit_per_task', 'concurrency'):
-            _cfg_dir = _P(__file__).parent / 'config'
+                bench_cfg.pop(k, None)
-            if not _cfg_dir.exists():
-                _cfg_dir = _P('/data1/sora/evalharness/EvalHarness/evalharness/config')
-            _cfg_name = getattr(args, 'config', '')
-            if not _cfg_name:
-                _yamls = sorted(_cfg_dir.glob('*.yaml')) if _cfg_dir.exists() else []
-                if len(_yamls) == 1:
-                    _cfg_name = _yamls[0].stem  # auto: the only config
-            if _cfg_name:
-                cfg_path = _cfg_dir / f'{_cfg_name}.yaml'
-                if cfg_path.exists():
-                    _all = _yaml.safe_load(open(cfg_path)) or {}
-                    _default = _all.get('default', {})
-                    bench_cfg = {**_default, **(_all.get(name) or {})}
-                    # strip non-generation keys (they go to run_eval kwargs)
-                    for k in ('judge', 'judge_url', 'env', 'max_turns',
-                              'limit', 'limit_per_task', 'concurrency'):
-                        bench_cfg.pop(k, None)
 
             # repeats: run the benchmark N times, report mean ± spread
             _repeats = int(bench_cfg.pop('repeats', 1) or 1)