Fix config auto-load silently disabled by the sample-counts manifest

Adding config/sample_counts.yaml made TWO yamls in config/, and the
auto-load rule was 'exactly one yaml' -- so every bench silently lost
its repeats/temperature/max_tokens (humaneval ran once instead of 3).
Manifest renamed to .json, and the rule hardened: a lone yaml still
wins, otherwise default.yaml wins explicitly. Verified: aime 12x +
humaneval 3x repeats active again.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -175,8 +175,14 @@ def _load_bench_cfg(args, name: str) -> dict:
     cfg_name = getattr(args, 'config', '')
     if not cfg_name and cfg_dir.exists():
         yamls = sorted(cfg_dir.glob('*.yaml'))
+        # auto-load: a lone config wins; otherwise default.yaml wins.
+        # (NB: non-config yaml sidecars must NOT land in config/ -- a
+        # sample-counts manifest here once disabled auto-load entirely and
+        # silently dropped every bench's repeats/temperature/max_tokens)
         if len(yamls) == 1:
-            cfg_name = yamls[0].stem  # auto: the only config
+            cfg_name = yamls[0].stem
+        elif any(y.stem == 'default' for y in yamls):
+            cfg_name = 'default'
     if not cfg_name:
         return {}
     cfg_path = cfg_dir / f'{cfg_name}.yaml'
@@ -202,12 +208,10 @@ def _plan_sample_counts(args):
     from evalharness.data import get_dataset
 
     manifest = {}
-    mf = Path(__file__).parent / 'config' / 'sample_counts.yaml'
+    mf = Path(__file__).parent / 'config' / 'sample_counts.json'
     if mf.exists():
         try:
-            import yaml as _yaml
+            manifest = json.load(open(mf)) or {}
-
-            manifest = _yaml.safe_load(open(mf)) or {}
         except Exception:
             manifest = {}
 

evalharness/config/sample_counts.json

@@ -0,0 +1,29 @@
+{
+ "humaneval": 164,
+ "aime24": 30,
+ "aime25": 30,
+ "aime26": 30,
+ "hmmt26": 30,
+ "gpqa_diamond": 198,
+ "mmlu": 285,
+ "mmlu_pro": 12032,
+ "cmmlu": 11528,
+ "gsm8k": 1319,
+ "arc": 2376,
+ "hellaswag": 10042,
+ "winogrande": 1267,
+ "drop": 9535,
+ "longbench_v2": 503,
+ "live_code_bench": 1055,
+ "bigcodebench": 1140,
+ "trivia_qa": 17944,
+ "simple_qa": 4326,
+ "hle": 2500,
+ "imo_answerbench": 89,
+ "openai_mrcr": 1300,
+ "bfcl_v3": 2000,
+ "general_fc": 400,
+ "tau2_bench": 184,
+ "competition_math": 5000,
+ "swe_bench_verified": 500
+}

evalharness/config/sample_counts.yaml

@@ -1,30 +0,0 @@
-# Full-dataset sample counts per benchmark (fallback for the Run Plan
-# when the dataset is not cached locally yet -- harvested from real runs;
-# update when a benchmark's source changes).
-humaneval: 164
-aime24: 30
-aime25: 30
-aime26: 30
-hmmt26: 30
-gpqa_diamond: 198
-mmlu: 285
-mmlu_pro: 12032
-cmmlu: 11528
-gsm8k: 1319
-arc: 2376
-hellaswag: 10042
-winogrande: 1267
-drop: 9535
-longbench_v2: 503
-live_code_bench: 1055
-bigcodebench: 1140
-trivia_qa: 17944
-simple_qa: 4326
-hle: 2500
-imo_answerbench: 89
-openai_mrcr: 1300
-bfcl_v3: 2000
-general_fc: 400
-tau2_bench: 184
-competition_math: 5000
-swe_bench_verified: 500