Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Config auto-loads: single yaml in config/ becomes default generation params

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 09:04:23 +00:00
parent fe1852302a
commit ed345c5ac1
1 changed files with 22 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
evalharness/cli.py
View File

@ -582,26 +582,30 @@ def _cmd_eval_run(args) -> int:
_shared_reporter.resume()
origin = ds.lineage.get('from', 'unknown')
_emit(f'Dataset ready: {sample_count} samples from {origin}')
# YAML config: per-bench generation params + extras
# YAML config: per-bench generation params, AUTO-LOADED
# (single .yaml in config/ = the default; --config overrides)
bench_cfg = {}
if getattr(args, 'config', ''):
import yaml as _yaml
from pathlib import Path as _P
import yaml as _yaml
from pathlib import Path as _P
cfg_path = (_P(__file__).parent / 'config' / f'{args.config}.yaml')
if not cfg_path.exists():
# fallback: source tree (dev mode, pip install -e)
cfg_path = _P('/data1/sora/evalharness/EvalHarness/evalharness/config') / f'{args.config}.yaml'
if not cfg_path.exists():
raise SystemExit(f'config not found: {args.config}.yaml')
_all = _yaml.safe_load(open(cfg_path)) or {}
_default = _all.get('default', {})
bench_cfg = {**_default, **(_all.get(name) or {})}
# strip non-generation keys (they go to run_eval kwargs)
for k in ('judge', 'judge_url', 'env', 'max_turns',
'limit', 'limit_per_task', 'concurrency', 'repeats'):
if k not in bench_cfg:
bench_cfg.pop(k, None) # no-op if absent
_cfg_dir = _P(__file__).parent / 'config'
if not _cfg_dir.exists():
_cfg_dir = _P('/data1/sora/evalharness/EvalHarness/evalharness/config')
_cfg_name = getattr(args, 'config', '')
if not _cfg_name:
_yamls = sorted(_cfg_dir.glob('*.yaml')) if _cfg_dir.exists() else []
if len(_yamls) == 1:
_cfg_name = _yamls[0].stem # auto: the only config
if _cfg_name:
cfg_path = _cfg_dir / f'{_cfg_name}.yaml'
if cfg_path.exists():
_all = _yaml.safe_load(open(cfg_path)) or {}
_default = _all.get('default', {})
bench_cfg = {**_default, **(_all.get(name) or {})}
# strip non-generation keys (they go to run_eval kwargs)
for k in ('judge', 'judge_url', 'env', 'max_turns',
'limit', 'limit_per_task', 'concurrency', 'repeats'):
bench_cfg.pop(k, None)
if model_spec: # generate + score in one go
from evalharness.model import run_eval