From fe1852302a42c4c5f26cc879343021c6c49f4c49 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 11 Sep 2026 09:00:28 +0000 Subject: [PATCH] --config flag loads evalharness/config/.yaml (per-bench generation params); yaml added to package-data; verified end-to-end Co-Authored-By: Claude --- evalharness/cli.py | 26 ++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 126b64e..db289db 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -582,6 +582,27 @@ def _cmd_eval_run(args) -> int: _shared_reporter.resume() origin = ds.lineage.get('from', 'unknown') _emit(f'Dataset ready: {sample_count} samples from {origin}') + # YAML config: per-bench generation params + extras + bench_cfg = {} + if getattr(args, 'config', ''): + import yaml as _yaml + from pathlib import Path as _P + + cfg_path = (_P(__file__).parent / 'config' / f'{args.config}.yaml') + if not cfg_path.exists(): + # fallback: source tree (dev mode, pip install -e) + cfg_path = _P('/data1/sora/evalharness/EvalHarness/evalharness/config') / f'{args.config}.yaml' + if not cfg_path.exists(): + raise SystemExit(f'config not found: {args.config}.yaml') + _all = _yaml.safe_load(open(cfg_path)) or {} + _default = _all.get('default', {}) + bench_cfg = {**_default, **(_all.get(name) or {})} + # strip non-generation keys (they go to run_eval kwargs) + for k in ('judge', 'judge_url', 'env', 'max_turns', + 'limit', 'limit_per_task', 'concurrency', 'repeats'): + if k not in bench_cfg: + bench_cfg.pop(k, None) # no-op if absent + if model_spec: # generate + score in one go from evalharness.model import run_eval @@ -626,9 +647,11 @@ def _cmd_eval_run(args) -> int: _reporter.set_phase('writing') else: _print_phase(_console, _idx, total_runs, _name, msg) + _gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})} report = asyncio.run(run_eval( ds, model_spec, concurrency=args.concurrency, limit=args.limit, limit_per_task=args.limit_per_task, + gen_kwargs=_gen_kw or None, checkpoint=args.resume, judge_spec=_compose_judge_spec(args), env=args.env, api_key=getattr(args, 'api_key', ''), @@ -927,6 +950,9 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--judge-provider', default='openai-chat', help='judge protocol/provider (default openai-chat; ' 'openai-pool for multi-endpoint judges)') + p.add_argument('--config', default='', + help='YAML config name (loads evalharness/config/.yaml ' + 'for per-bench generation params + repeats)') p.add_argument('--profile', default='', help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short ' 'or any @register_gen_profile name); layers: plugin default < ' diff --git a/pyproject.toml b/pyproject.toml index ab1c436..77e5ebf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,4 +35,4 @@ evalharness = "evalharness.cli:main" include = ["evalharness*"] [tool.setuptools.package-data] -"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv", "*.txt", "*.md", "references/*.json"] +"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv", "*.txt", "*.md", "*.yaml", "references/*.json"]