Run plan: concrete sample counts + 'auto' concurrency display/alias

- sample-counts manifest (config/sample_counts.yaml, harvested from
  real runs): uncached benches still show exact numbers in the plan
  instead of 'counts when datasets load' -- 'cache+est.' marks the mix
- '--concurrency auto' is now an alias for --auto-concurrency
- Concurrency row shows 'auto (start 8, gate decides)' when the gate
  drives, instead of a bare misleading 8

Also verified end-to-end: thinking-mode humaneval rep1/rep2 both
pass 98.8%, matching the es reference runs (98.17/98.78/98.78) on the
same model -- framework alignment holds on the thinking path too.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -193,25 +193,38 @@ def _load_bench_cfg(args, name: str) -> dict:
 
 def _plan_sample_counts(args):
     """(total_samples, total_generations, n_uncached) across the planned
-    benches, counted from LOCAL cache entries only -- never touches the
+    benches. Cache-first (exact); uncached benches fall back to the shipped
-    network, so the run plan stays instant on cold machines. Uncached
+    sample-counts manifest so the plan still shows CONCRETE numbers on a
-    benches simply don't contribute yet.
+    cold machine instead of 'counts when datasets load'.
     """
+    from pathlib import Path
+
     from evalharness.data import get_dataset
 
+    manifest = {}
+    mf = Path(__file__).parent / 'config' / 'sample_counts.yaml'
+    if mf.exists():
+        try:
+            import yaml as _yaml
+
+            manifest = _yaml.safe_load(open(mf)) or {}
+        except Exception:
+            manifest = {}
+
     total = gens = uncached = 0
     for name in getattr(args, 'datasets', []) or []:
+        n = None
         try:
             ds = get_dataset(name, **_overrides(args))
             cache_file = ds.cache_dir / 'samples.jsonl'
-            if not cache_file.exists():
+            if cache_file.exists():
-                uncached += 1
+                with open(cache_file, 'rb') as f:
-                continue
+                    n = sum(1 for _ in f)
-            with open(cache_file, 'rb') as f:
-                n = sum(1 for _ in f)
         except Exception:
+            n = None
+        if n is None:
             uncached += 1
-            continue
+            n = int(manifest.get(name, 0) or 0)  # estimate; 0 = unknown
         if getattr(args, 'limit', None):
             n = min(n, args.limit)
         total += n
@@ -232,21 +245,23 @@ def _print_run_plan(console, args, model_spec):
         cap = f' · ≤{args.limit} per bench (--limit)'
     elif getattr(args, 'limit_per_task', None):
         cap = f' · ≤{args.limit_per_task} per subject (--limit-per-task)'
+    src = 'cached' if not n_uncached else ('cache+est.' if n_samples else 'est.')
     if n_samples:
-        samples = f'{n_samples:,} samples (cached){cap}'
+        samples = f'{n_samples:,} samples ({src}){cap}'
         if n_gens > n_samples:  # repeats multiply the real work
-            samples = (f'{n_samples:,} samples (cached){cap} → '
+            samples = (f'{n_samples:,} samples ({src}){cap} → '
                        f'{n_gens:,} generations (repeats)')
-        if n_uncached:
+    elif getattr(args, 'limit', None):
-            samples += f' · {n_uncached} bench(es) not cached yet'
+        samples = f'up to {args.limit} per bench (--limit)'
-    elif n_uncached:
+    elif getattr(args, 'limit_per_task', None):
-        if getattr(args, 'limit', None):
+        samples = f'up to {args.limit_per_task} per subject (--limit-per-task)'
-            samples = f'up to {args.limit} per bench (--limit), counts when datasets load'
+    else:
-        elif getattr(args, 'limit_per_task', None):
+        samples = '? (unknown benchmarks)'
-            samples = (f'up to {args.limit_per_task} per subject '
+    if n_uncached:
-                       '(--limit-per-task), counts when datasets load')
+        samples += f' · {n_uncached} bench(es) not cached yet'
-        else:
+    _auto = getattr(args, 'auto_concurrency', False)
-            samples = 'full dataset, counts when datasets load (none cached yet)'
+    _conc = (f'auto (start {args.concurrency}, gate decides)' if _auto
+             else str(args.concurrency))
     if console is None:
         print(f'=== {title} ===')
         print(f'Provider: {provider}')
@@ -254,7 +269,7 @@ def _print_run_plan(console, args, model_spec):
         print(f'Model: {model_name}')
         print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}')
         print(f'Samples: {samples}')
-        print(f'Concurrency: {args.concurrency} | Thinking: '
+        print(f'Concurrency: {_conc} | Thinking: '
               f'{"enabled" if not args.disable_thinking else "disabled"} | '
               f'Performance: {"on" if args.perf else "off"}')
         print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}')
@@ -271,7 +286,7 @@ def _print_run_plan(console, args, model_spec):
     table.add_row('Model', model_name)
     table.add_row('Benchmarks', f'{len(args.datasets)}  ·  {", ".join(args.datasets)}')
     table.add_row('Samples', samples)
-    table.add_row('Concurrency', str(args.concurrency))
+    table.add_row('Concurrency', f'[magenta]{_conc}[/magenta]' if _auto else _conc)
     table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]')
     table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')
     table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]')
@@ -952,6 +967,11 @@ def _cmd_eval_run(args) -> int:
                          'secs': round(_time.time() - t0, 1), 'ok': False,
                          'err': f'{type(e).__name__}: {str(e)[:100]}'})
             print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)
+            import traceback as _tb
+
+            _tb.print_exc()   # full stack to stderr: the one-line form hides
+            # WHERE an error like 'Object of type ellipsis is not JSON
+            # serializable' actually comes from
             from evalharness.hooks import fire as _fire
 
             _fire('on_benchmark_failed', name=name, error=e, dataset=name)
@@ -1160,7 +1180,10 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument('--textools', action='store_true',
                    help='send tools as text instead of native tool calls')
     p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
-    p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
+    p.add_argument('--concurrency', default='32',
+                   help="parallel model calls (default 32); 'auto' = same as "
+                        "--auto-concurrency (the adaptive gate decides, this "
+                        "value becomes its starting point)")
     p.add_argument('--progress', action='store_true', default=True,
                    help='show per-sample progress (default: on)')
     p.add_argument('--no-progress', dest='progress', action='store_false',
@@ -1225,6 +1248,14 @@ def build_parser() -> argparse.ArgumentParser:
 
 def main(argv=None) -> int:
     args = build_parser().parse_args(argv)
+    # '--concurrency auto' == '--auto-concurrency': normalize once, here,
+    # so every downstream site (plan display, run_eval, gate initial) sees
+    # an int + the flag
+    if str(getattr(args, 'concurrency', '32')).strip().lower() == 'auto':
+        args.auto_concurrency = True
+        args.concurrency = 8
+    else:
+        args.concurrency = int(args.concurrency)
     return args.func(args)
 
 

evalharness/config/sample_counts.yaml

@@ -0,0 +1,30 @@
+# Full-dataset sample counts per benchmark (fallback for the Run Plan
+# when the dataset is not cached locally yet -- harvested from real runs;
+# update when a benchmark's source changes).
+humaneval: 164
+aime24: 30
+aime25: 30
+aime26: 30
+hmmt26: 30
+gpqa_diamond: 198
+mmlu: 285
+mmlu_pro: 12032
+cmmlu: 11528
+gsm8k: 1319
+arc: 2376
+hellaswag: 10042
+winogrande: 1267
+drop: 9535
+longbench_v2: 503
+live_code_bench: 1055
+bigcodebench: 1140
+trivia_qa: 17944
+simple_qa: 4326
+hle: 2500
+imo_answerbench: 89
+openai_mrcr: 1300
+bfcl_v3: 2000
+general_fc: 400
+tau2_bench: 184
+competition_math: 5000
+swe_bench_verified: 500