From ed345c5ac1018dd2d159712f83ec08186e19908a Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Fri, 11 Sep 2026 09:04:23 +0000
Subject: [PATCH] Config auto-loads: single yaml in config/ becomes default
 generation params

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/cli.py | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/evalharness/cli.py b/evalharness/cli.py
index db289db..56363fb 100644
--- a/evalharness/cli.py
+++ b/evalharness/cli.py
@@ -582,26 +582,30 @@ def _cmd_eval_run(args) -> int:
                 _shared_reporter.resume()
             origin = ds.lineage.get('from', 'unknown')
             _emit(f'Dataset ready: {sample_count} samples from {origin}')
-            # YAML config: per-bench generation params + extras
+            # YAML config: per-bench generation params, AUTO-LOADED
+            # (single .yaml in config/ = the default; --config overrides)
             bench_cfg = {}
-            if getattr(args, 'config', ''):
-                import yaml as _yaml
-                from pathlib import Path as _P
+            import yaml as _yaml
+            from pathlib import Path as _P
 
-                cfg_path = (_P(__file__).parent / 'config' / f'{args.config}.yaml')
-                if not cfg_path.exists():
-                    # fallback: source tree (dev mode, pip install -e)
-                    cfg_path = _P('/data1/sora/evalharness/EvalHarness/evalharness/config') / f'{args.config}.yaml'
-                if not cfg_path.exists():
-                    raise SystemExit(f'config not found: {args.config}.yaml')
-                _all = _yaml.safe_load(open(cfg_path)) or {}
-                _default = _all.get('default', {})
-                bench_cfg = {**_default, **(_all.get(name) or {})}
-                # strip non-generation keys (they go to run_eval kwargs)
-                for k in ('judge', 'judge_url', 'env', 'max_turns',
-                          'limit', 'limit_per_task', 'concurrency', 'repeats'):
-                    if k not in bench_cfg:
-                        bench_cfg.pop(k, None)  # no-op if absent
+            _cfg_dir = _P(__file__).parent / 'config'
+            if not _cfg_dir.exists():
+                _cfg_dir = _P('/data1/sora/evalharness/EvalHarness/evalharness/config')
+            _cfg_name = getattr(args, 'config', '')
+            if not _cfg_name:
+                _yamls = sorted(_cfg_dir.glob('*.yaml')) if _cfg_dir.exists() else []
+                if len(_yamls) == 1:
+                    _cfg_name = _yamls[0].stem  # auto: the only config
+            if _cfg_name:
+                cfg_path = _cfg_dir / f'{_cfg_name}.yaml'
+                if cfg_path.exists():
+                    _all = _yaml.safe_load(open(cfg_path)) or {}
+                    _default = _all.get('default', {})
+                    bench_cfg = {**_default, **(_all.get(name) or {})}
+                    # strip non-generation keys (they go to run_eval kwargs)
+                    for k in ('judge', 'judge_url', 'env', 'max_turns',
+                              'limit', 'limit_per_task', 'concurrency', 'repeats'):
+                        bench_cfg.pop(k, None)
 
             if model_spec:  # generate + score in one go
                 from evalharness.model import run_eval