--config flag loads evalharness/config/<name>.yaml (per-bench generation params); yaml added to package-data; verified end-to-end

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -582,6 +582,27 @@ def _cmd_eval_run(args) -> int:
                 _shared_reporter.resume()
             origin = ds.lineage.get('from', 'unknown')
             _emit(f'Dataset ready: {sample_count} samples from {origin}')
+            # YAML config: per-bench generation params + extras
+            bench_cfg = {}
+            if getattr(args, 'config', ''):
+                import yaml as _yaml
+                from pathlib import Path as _P
+
+                cfg_path = (_P(__file__).parent / 'config' / f'{args.config}.yaml')
+                if not cfg_path.exists():
+                    # fallback: source tree (dev mode, pip install -e)
+                    cfg_path = _P('/data1/sora/evalharness/EvalHarness/evalharness/config') / f'{args.config}.yaml'
+                if not cfg_path.exists():
+                    raise SystemExit(f'config not found: {args.config}.yaml')
+                _all = _yaml.safe_load(open(cfg_path)) or {}
+                _default = _all.get('default', {})
+                bench_cfg = {**_default, **(_all.get(name) or {})}
+                # strip non-generation keys (they go to run_eval kwargs)
+                for k in ('judge', 'judge_url', 'env', 'max_turns',
+                          'limit', 'limit_per_task', 'concurrency', 'repeats'):
+                    if k not in bench_cfg:
+                        bench_cfg.pop(k, None)  # no-op if absent
+
             if model_spec:  # generate + score in one go
                 from evalharness.model import run_eval
 
@@ -626,9 +647,11 @@ def _cmd_eval_run(args) -> int:
                             _reporter.set_phase('writing')
                     else:
                         _print_phase(_console, _idx, total_runs, _name, msg)
+                _gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})}
                 report = asyncio.run(run_eval(
                     ds, model_spec, concurrency=args.concurrency, limit=args.limit,
                     limit_per_task=args.limit_per_task,
+                    gen_kwargs=_gen_kw or None,
                     checkpoint=args.resume,
                     judge_spec=_compose_judge_spec(args), env=args.env,
                     api_key=getattr(args, 'api_key', ''),
@@ -927,6 +950,9 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument('--judge-provider', default='openai-chat',
                    help='judge protocol/provider (default openai-chat; '
                         'openai-pool for multi-endpoint judges)')
+    p.add_argument('--config', default='',
+                   help='YAML config name (loads evalharness/config/<name>.yaml '
+                        'for per-bench generation params + repeats)')
     p.add_argument('--profile', default='',
                    help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short '
                         'or any @register_gen_profile name); layers: plugin default < '

pyproject.toml

@@ -35,4 +35,4 @@ evalharness = "evalharness.cli:main"
 include = ["evalharness*"]
 
 [tool.setuptools.package-data]
-"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv", "*.txt", "*.md", "references/*.json"]
+"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv", "*.txt", "*.md", "*.yaml", "references/*.json"]