From f6ee6c7a8b886ff567479c22c1c4431d3b3fbf82 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Mon, 14 Sep 2026 09:23:41 +0000
Subject: [PATCH] Fix config auto-load silently disabled by the sample-counts
 manifest

Adding config/sample_counts.yaml made TWO yamls in config/, and the
auto-load rule was 'exactly one yaml' -- so every bench silently lost
its repeats/temperature/max_tokens (humaneval ran once instead of 3).
Manifest renamed to .json, and the rule hardened: a lone yaml still
wins, otherwise default.yaml wins explicitly. Verified: aime 12x +
humaneval 3x repeats active again.

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/cli.py                    | 14 ++++++++-----
 evalharness/config/sample_counts.json | 29 ++++++++++++++++++++++++++
 evalharness/config/sample_counts.yaml | 30 ---------------------------
 3 files changed, 38 insertions(+), 35 deletions(-)
 create mode 100644 evalharness/config/sample_counts.json
 delete mode 100644 evalharness/config/sample_counts.yaml

diff --git a/evalharness/cli.py b/evalharness/cli.py
index a7b1958..d351e29 100644
--- a/evalharness/cli.py
+++ b/evalharness/cli.py
@@ -175,8 +175,14 @@ def _load_bench_cfg(args, name: str) -> dict:
     cfg_name = getattr(args, 'config', '')
     if not cfg_name and cfg_dir.exists():
         yamls = sorted(cfg_dir.glob('*.yaml'))
+        # auto-load: a lone config wins; otherwise default.yaml wins.
+        # (NB: non-config yaml sidecars must NOT land in config/ -- a
+        # sample-counts manifest here once disabled auto-load entirely and
+        # silently dropped every bench's repeats/temperature/max_tokens)
         if len(yamls) == 1:
-            cfg_name = yamls[0].stem  # auto: the only config
+            cfg_name = yamls[0].stem
+        elif any(y.stem == 'default' for y in yamls):
+            cfg_name = 'default'
     if not cfg_name:
         return {}
     cfg_path = cfg_dir / f'{cfg_name}.yaml'
@@ -202,12 +208,10 @@ def _plan_sample_counts(args):
     from evalharness.data import get_dataset
 
     manifest = {}
-    mf = Path(__file__).parent / 'config' / 'sample_counts.yaml'
+    mf = Path(__file__).parent / 'config' / 'sample_counts.json'
     if mf.exists():
         try:
-            import yaml as _yaml
-
-            manifest = _yaml.safe_load(open(mf)) or {}
+            manifest = json.load(open(mf)) or {}
         except Exception:
             manifest = {}
 
diff --git a/evalharness/config/sample_counts.json b/evalharness/config/sample_counts.json
new file mode 100644
index 0000000..7d0705f
--- /dev/null
+++ b/evalharness/config/sample_counts.json
@@ -0,0 +1,29 @@
+{
+ "humaneval": 164,
+ "aime24": 30,
+ "aime25": 30,
+ "aime26": 30,
+ "hmmt26": 30,
+ "gpqa_diamond": 198,
+ "mmlu": 285,
+ "mmlu_pro": 12032,
+ "cmmlu": 11528,
+ "gsm8k": 1319,
+ "arc": 2376,
+ "hellaswag": 10042,
+ "winogrande": 1267,
+ "drop": 9535,
+ "longbench_v2": 503,
+ "live_code_bench": 1055,
+ "bigcodebench": 1140,
+ "trivia_qa": 17944,
+ "simple_qa": 4326,
+ "hle": 2500,
+ "imo_answerbench": 89,
+ "openai_mrcr": 1300,
+ "bfcl_v3": 2000,
+ "general_fc": 400,
+ "tau2_bench": 184,
+ "competition_math": 5000,
+ "swe_bench_verified": 500
+}
\ No newline at end of file
diff --git a/evalharness/config/sample_counts.yaml b/evalharness/config/sample_counts.yaml
deleted file mode 100644
index 10d2818..0000000
--- a/evalharness/config/sample_counts.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Full-dataset sample counts per benchmark (fallback for the Run Plan
-# when the dataset is not cached locally yet -- harvested from real runs;
-# update when a benchmark's source changes).
-humaneval: 164
-aime24: 30
-aime25: 30
-aime26: 30
-hmmt26: 30
-gpqa_diamond: 198
-mmlu: 285
-mmlu_pro: 12032
-cmmlu: 11528
-gsm8k: 1319
-arc: 2376
-hellaswag: 10042
-winogrande: 1267
-drop: 9535
-longbench_v2: 503
-live_code_bench: 1055
-bigcodebench: 1140
-trivia_qa: 17944
-simple_qa: 4326
-hle: 2500
-imo_answerbench: 89
-openai_mrcr: 1300
-bfcl_v3: 2000
-general_fc: 400
-tau2_bench: 184
-competition_math: 5000
-swe_bench_verified: 500