From 4a15f80897311a1d02d4f9b351ce1c64a860a4ba Mon Sep 17 00:00:00 2001 From: sora Date: Mon, 24 Aug 2026 06:09:38 +0000 Subject: [PATCH] Add evaluation layer + visualization: extract/score/aggregate plugins, 28 recipes, official-aligned scorers (PRM800K math, DROP Hungarian EM/F1, SimpleQA A/B/C judge), report artifacts, console renderers, CLI eval/viz, regression tests --- README.md | 122 +++++-- evalharness/cli.py | 60 ++++ evalharness/data/dataset.py | 4 + evalharness/data/datasets/aime26.py | 10 +- evalharness/data/datasets/bfcl_v3.py | 8 +- evalharness/data/datasets/gpqa_diamond.py | 8 +- evalharness/data/datasets/hmmt26.py | 8 +- evalharness/data/datasets/imo_answerbench.py | 6 +- evalharness/data/datasets/live_code_bench.py | 8 +- evalharness/data/datasets/openai_mrcr.py | 6 +- evalharness/data/datasets/tau2_bench.py | 6 +- evalharness/data/loader.py | 125 ++++++- evalharness/eval/__init__.py | 47 +++ evalharness/eval/aggregator.py | 124 +++++++ evalharness/eval/extractor.py | 212 ++++++++++++ evalharness/eval/math_grader.py | 145 +++++++++ evalharness/eval/recipe.py | 82 +++++ evalharness/eval/recipes/__init__.py | 5 + evalharness/eval/recipes/agent.py | 78 +++++ evalharness/eval/recipes/judged.py | 60 ++++ evalharness/eval/recipes/math.py | 50 +++ evalharness/eval/recipes/mcq.py | 53 +++ evalharness/eval/recipes/qa.py | 61 ++++ evalharness/eval/record.py | 69 ++++ evalharness/eval/registry.py | 34 ++ evalharness/eval/runner.py | 125 +++++++ evalharness/eval/scorer.py | 323 +++++++++++++++++++ evalharness/viz/__init__.py | 63 ++++ evalharness/viz/renderers/__init__.py | 0 evalharness/viz/renderers/text.py | 145 +++++++++ pyproject.toml | 6 +- tests/test_eval.py | 154 +++++++++ 32 files changed, 2147 insertions(+), 60 deletions(-) create mode 100644 evalharness/eval/__init__.py create mode 100644 evalharness/eval/aggregator.py create mode 100644 evalharness/eval/extractor.py create mode 100644 evalharness/eval/math_grader.py create mode 100644 evalharness/eval/recipe.py create mode 100644 evalharness/eval/recipes/__init__.py create mode 100644 evalharness/eval/recipes/agent.py create mode 100644 evalharness/eval/recipes/judged.py create mode 100644 evalharness/eval/recipes/math.py create mode 100644 evalharness/eval/recipes/mcq.py create mode 100644 evalharness/eval/recipes/qa.py create mode 100644 evalharness/eval/record.py create mode 100644 evalharness/eval/registry.py create mode 100644 evalharness/eval/runner.py create mode 100644 evalharness/eval/scorer.py create mode 100644 evalharness/viz/__init__.py create mode 100644 evalharness/viz/renderers/__init__.py create mode 100644 evalharness/viz/renderers/text.py create mode 100644 tests/test_eval.py diff --git a/README.md b/README.md index 5a5bbd4..82c63fd 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # EvalHarness -A plugin-based LLM/agent evaluation harness. **Currently: the data layer only** -(dataset registration + lazy materialization cache + CLI). Model/eval/sandbox/ -tool/skill layers are intentionally not built yet — they land one at a time. +A plugin-based LLM/agent evaluation harness. **Currently: the data layer + +the evaluation layer** (datasets & eval recipes as plugins, lazy +materialization cache, official-aligned scorers, report artifacts & console +visualization). Sandbox/model/tool/skill layers land one at a time. ## Features @@ -27,23 +28,19 @@ tool/skill layers are intentionally not built yet — they land one at a time. ## Install ```bash -pip install . # from this repo; core only (pydantic) -pip install '.[hub]' # + HuggingFace `datasets`, needed for hub sources +pip install . # core only (pydantic) +pip install '.[hub]' # + HuggingFace datasets +pip install '.[math]' # + sympy/pylatexenc (aime/math official grader) +pip install '.[exec]' # + numpy/scipy (official DROP aligner) +pip install '.[hub,parquet,math,exec]' # everything ``` -For development: `pip install -e '.[hub]'`. +For development: `pip install -e '.[hub,parquet,math,exec]'`. -To build and install the wheel yourself: - -```bash -pip install build -python -m build # produces dist/evalharness-0.1.0-py3-none-any.whl + .tar.gz -pip install dist/evalharness-0.1.0-py3-none-any.whl -``` - -> Publishing to PyPI (`twine upload dist/*`) makes `pip install evalharness` -> work for everyone — note the name may be taken, so pick a unique -> distribution name (e.g. `evalharness-x`) in `pyproject.toml` before upload. +Dependency tiers (deliberate — no monolithic image): text-compare recipes +run on core; symbolic math needs `[math]`; DROP needs `[exec]`; code-execution +benchmarks (humaneval/bigcodebench/LCB/bfcl) additionally require the sandbox +layer (roadmap) since they run model-generated code in isolation. ## Quick start @@ -110,6 +107,46 @@ datasets/ > different sources (`--source`) or params would otherwise collide and serve > stale data. The short hash keeps them apart while staying readable. +## Evaluation layer + +```bash +evalharness eval list # 28 recipes, per benchmark +evalharness eval run gsm8k preds.jsonl --model mymodel --out gsm8k.report.json +evalharness viz show gsm8k.report.json # console table +evalharness viz show r1.json r2.json --style md_compare +``` + +```python +from evalharness import get_dataset +from evalharness.eval import evaluate, get_eval +from evalharness.viz import render + +ds = get_dataset('gsm8k') +report = evaluate(ds, predictions, model='mymodel') # recipe auto-resolved +report.save('gsm8k.report.json') +print(render(report, style='text')) +``` + +Pipeline: **extract -> score -> aggregate**, each stage a registered plugin: + +- **Extractors** (`eval/extractor.py`): `math_boxed / mcq_letter / + answer_phrase / answer_spans / gsm8k_hash / last_number / code_block / + quoted_list / identity` + cascades (first stage that succeeds wins). +- **Scorers** (`eval/scorer.py`): text compare (`exact / math_equal / + em_f1 / alias_match` — official implementations), `llm_judge` (labels-> + scores contract, wired to a ModelAdapter via `evaluate(judge=...)`), + `execution` / `env_reward` (raise LayerNotReady until sandbox/agent land). +- **Aggregators** (`eval/aggregator.py`): `mean / pass_at_k / + grouped_avg / weighted_group_avg / binned_avg / simpleqa_official` + (is_given_attempted + accuracy_given_attempted). +- **Recipes** (`eval/recipes/`): one per benchmark, 5-20 lines each — pure + bindings of the primitives above, `@register_eval('gsm8k')`. + +Per-sample results keep `raw_prediction` + extraction note + score details; +`extraction_failure_rate` is reported as a health metric. Changing a recipe +and re-running `evaluate()` re-scores the same predictions — the model is +never re-queried. + ## Built-in datasets (28, official sources) | Family | Datasets (source) | @@ -168,6 +205,14 @@ The data layer mirrors conclusions from a close reading of seven evaluation frameworks (evalscope, lm-evaluation-harness, inspect_ai, deepeval, VLMEvalKit, harbor, deepseek-harness); see `../README.md` for the full analysis: +- **Data layer vs eval layer: strict separation** (unlike evalscope's + DataAdapter, which welds record conversion + extraction + scoring + prompt + into one class). The data layer answers *what is the question/target*; the + eval layer answers *how to judge a response*. Consequences: predictions can + be re-scored under a new recipe without re-running the model; extraction + rules are shared primitives (`math_boxed`, `mcq_letter`, ...) instead of + per-benchmark copies; official grader fixes (e.g. DROP's Hungarian + alignment) land in one scorer and apply to every recipe. - **`Sample` base + one `record_to_sample` per plugin** (evalscope/inspect_ai): raw formats vary wildly; the unified format only exists after conversion, and everything downstream sees only `Sample`. @@ -178,6 +223,13 @@ harbor, deepseek-harness); see `../README.md` for the full analysis: metadata + conversion recipes; `list` never touches the network. - **Config-sensitive cache keys** (evalscope): a cache hit is always correct; no invalidation logic exists. +- **Raw predictions are immutable artifacts**: every SampleResult keeps + raw_prediction; extraction/score details are re-derivable. Failed + extractions are surfaced (extraction_failure_rate), never silently zeroed. +- **Official grader parity**: scorers replicate official logic where it + exists (PRM800K math equivalence; DROP's answer_to_bags + + linear_sum_assignment; SimpleQA's A/B/C judge with NOT_ATTEMPTED fallback + and accuracy_given_attempted). Official test cases anchor `tests/`. - **Sandbox/tool fields on `Sample` are declarations only** (`sandbox/files/setup/tools`): the data layer never executes; a future sandbox layer will materialize them. @@ -196,22 +248,38 @@ harbor, deepseek-harness); see `../README.md` for the full analysis: EvalHarness/ ├── pyproject.toml ├── evalharness/ -│ ├── cli.py # argparse CLI: data list/fetch(concurrent)/stats/show -│ └── data/ -│ ├── sample.py # Sample / ChatMessage / SandboxSpec / ToolInfo -│ ├── spec.py # DatasetSpec (metadata) / FieldSpec (field mapping) -│ ├── registry.py # Registry + @register_dataset + get_dataset -│ ├── loader.py # raw record loading (local file/dir/hub) -│ ├── dataset.py # Dataset: lazy materialize + cache + derived views -│ └── datasets/ # 28 built-in single-file dataset plugins +│ ├── cli.py # CLI: data list/fetch/unload/stats/show +│ │ # eval list/run, viz show +│ ├── data/ # ---- data layer ---- +│ │ ├── sample.py # Sample / ChatMessage / SandboxSpec / ToolInfo +│ │ ├── spec.py # DatasetSpec (metadata) / FieldSpec (field mapping) +│ │ ├── registry.py # Registry + @register_dataset + get_dataset +│ │ ├── loader.py # raw loading (local/HF/ModelScope native) +│ │ ├── dataset.py # Dataset: lazy materialize + cache + derived views +│ │ └── datasets/ # 28 built-in single-file dataset plugins +│ ├── eval/ # ---- evaluation layer ---- +│ │ ├── record.py # SampleResult / EvalReport artifacts +│ │ ├── extractor.py # answer-extraction primitives (+cascades) +│ │ ├── scorer.py # scoring primitives (official implementations) +│ │ ├── math_grader.py # PRM800K sympy equivalence (optional [math]) +│ │ ├── aggregator.py # mean/pass@k/grouped/binned/simpleqa_official +│ │ ├── recipe.py # EvalRecipe bindings + @register_eval +│ │ ├── runner.py # evaluate(dataset, predictions) -> EvalReport +│ │ └── recipes/ # 28 per-benchmark recipes +│ └── viz/ # ---- visualization (report consumer) ---- +│ └── renderers/text.py # text / md / md_compare / radar / errors ├── examples/ │ └── data/ # offline demo subsets (gsm8k/cmmlu, 5 rows each) +└── tests/ + └── test_eval.py # official-anchor regression tests ``` ## Roadmap (not built yet, one layer at a time) -- [ ] Model layer (ModelAdapter: unified URL-based invocation) -- [ ] Evaluation engine (evaluator + scorer/metric) +- [x] Data layer (28 dataset plugins, lazy cache, native HF/ModelScope loaders) +- [x] Evaluation layer (extract/score/aggregate plugins, official scorers, recipes) +- [x] Visualization (console/markdown renderers over report artifacts) +- [ ] Model layer (ModelAdapter: unified URL-based invocation; wires llm_judge) - [ ] Sandbox layer (materialize `Sample.sandbox`: lazy per-instance image pull, refcounted image unload, container lifecycle; `requires` gating) - [ ] Tool layer (data filter/synthesis/dedup/export; Dataset in, Dataset out) diff --git a/evalharness/cli.py b/evalharness/cli.py index 300f9a2..b789ce1 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -98,6 +98,39 @@ def _add_override_flags(p: argparse.ArgumentParser) -> None: p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)') +def _cmd_eval_list(_args) -> int: + from evalharness.eval import list_evals + + names = list_evals() + print('\n'.join(names) if names else 'no eval recipes registered') + print(f'\n{len(names)} eval recipe(s) registered') + return 0 + + +def _cmd_eval_run(args) -> int: + from evalharness.eval import evaluate, get_eval + from evalharness.data import get_dataset + + ds = get_dataset(args.dataset, **_overrides(args)) + preds = [json.loads(line) for line in open(args.predictions, encoding='utf-8') if line.strip()] + preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p for p in preds] + report = evaluate(ds, preds, model=args.model) + if args.out: + report.save(args.out) + print(f'saved -> {args.out}') + from evalharness.viz import render + + print(render(report, style=args.style)) + return 0 + + +def _cmd_viz_show(args) -> int: + from evalharness.viz import render + + print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {}))) + return 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI') sub = parser.add_subparsers(dest='command', required=True) @@ -131,6 +164,33 @@ def build_parser() -> argparse.ArgumentParser: _add_override_flags(p) p.set_defaults(func=_cmd_data_show) + # ---- eval ---- + ev = sub.add_parser('eval', help='evaluation recipes & runs') + esub = ev.add_subparsers(dest='eval_command', required=True) + + p = esub.add_parser('list', help='list registered eval recipes') + p.set_defaults(func=_cmd_eval_list) + + p = esub.add_parser('run', help='score predictions against a dataset') + p.add_argument('dataset', help='dataset name (recipe auto-resolved)') + p.add_argument('predictions', help='jsonl: one raw string or {"raw": ...} per sample') + p.add_argument('--model', default='', help='model tag recorded in the report') + p.add_argument('--out', help='save the EvalReport json here') + p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)') + _add_override_flags(p) + p.set_defaults(func=_cmd_eval_run) + + # ---- viz ---- + vz = sub.add_parser('viz', help='render saved EvalReport artifacts') + zsub = vz.add_subparsers(dest='viz_command', required=True) + + p = zsub.add_parser('show', help='render report file(s)') + p.add_argument('reports', nargs='+') + p.add_argument('--style', default='text', + help='text | md | md_compare (multi) | radar | errors') + p.add_argument('-n', type=int, help='for errors style: how many samples') + p.set_defaults(func=_cmd_viz_show) + return parser diff --git a/evalharness/data/dataset.py b/evalharness/data/dataset.py index 057f74e..134a267 100644 --- a/evalharness/data/dataset.py +++ b/evalharness/data/dataset.py @@ -120,6 +120,10 @@ class Dataset: sample = self._record_fn(record) if not sample.task_type: sample.task_type = self.spec.task_type + # expose the loaded subset on every sample: per-subtask eval dispatch + # (e.g. bbh MC vs free-form) reads metadata['subset'] + if self.spec.subset not in ('default', 'all') and sample.metadata.get('subset') is None: + sample.metadata.setdefault('subset', self.spec.subset) return sample @staticmethod diff --git a/evalharness/data/datasets/aime26.py b/evalharness/data/datasets/aime26.py index 6ad41ed..277abcf 100644 --- a/evalharness/data/datasets/aime26.py +++ b/evalharness/data/datasets/aime26.py @@ -1,4 +1,4 @@ -"""AIME 2026. No official standalone release; community-curated (evalscope).""" +"""AIME 2026. No official standalone release; MathArena community curation.""" from ..sample import Sample from ..registry import register_dataset @@ -8,12 +8,12 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='aime26', - source='evalscope/aime26', # curated; no official upstream release (ModelScope) - split='test', + source='MathArena/aime_2026', # curated by MathArena (HuggingFace) + split='train', # the dataset ships a single split task_type='math', tags=['math', 'competition'], - description='AIME 2026 (community-curated, no official upstream).', - params={'hub': 'modelscope', 'ms_files': ['aime2026.jsonl']}, + description='AIME 2026, 30 problems (integer answers 000-999).', + params={'hub': 'hf_raw'}, ) ) def aime26(): diff --git a/evalharness/data/datasets/bfcl_v3.py b/evalharness/data/datasets/bfcl_v3.py index ac9607f..b303a96 100644 --- a/evalharness/data/datasets/bfcl_v3.py +++ b/evalharness/data/datasets/bfcl_v3.py @@ -17,12 +17,12 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='bfcl_v3', - source='AI-ModelScope/bfcl_v3', # ModelScope mirror of the official GitHub data - split='train', # the mirror ships a single split + source='llamastack/bfcl_v3', # HF conversion of the official GitHub data + split='train', # the conversion ships a single split task_type='fc', tags=['function_calling', 'tool_use'], - description='BFCL v3 function calling (official content, ModelScope mirror).', - params={'hub': 'modelscope'}, + description='BFCL v3 function calling (official content, HF conversion).', + params={'hub': 'hf_raw'}, ) ) def bfcl_v3(): diff --git a/evalharness/data/datasets/gpqa_diamond.py b/evalharness/data/datasets/gpqa_diamond.py index a7fa9d2..53fa114 100644 --- a/evalharness/data/datasets/gpqa_diamond.py +++ b/evalharness/data/datasets/gpqa_diamond.py @@ -1,8 +1,8 @@ """GPQA Diamond. Official source Idavidrein/gpqa is gated on HF (needs auth); we load the -ModelScope mirror (AI-ModelScope/gpqa_diamond) which keeps the official -column layout (Question / Correct Answer / Incorrect Answer 1-3). +nmayorga7 CSV export which keeps the official column layout (Question / +Correct Answer / Incorrect Answer 1-3). Note: choices are stored with the correct answer first (target 'A'). Option shuffling is an eval-time concern (the evaluator should shuffle @@ -17,12 +17,12 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='gpqa_diamond', - source='AI-ModelScope/gpqa_diamond', # ModelScope mirror of the gated official + source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official split='train', task_type='mcq', tags=['knowledge', 'science'], description='GPQA diamond split, graduate-level science MCQ (official content).', - params={'hub': 'modelscope'}, + params={'hub': 'hf_raw'}, ) ) def gpqa_diamond(): diff --git a/evalharness/data/datasets/hmmt26.py b/evalharness/data/datasets/hmmt26.py index 022ac4a..b6ca7b1 100644 --- a/evalharness/data/datasets/hmmt26.py +++ b/evalharness/data/datasets/hmmt26.py @@ -1,4 +1,4 @@ -"""HMMT Feb 2026. No official standalone release; community-curated (evalscope).""" +"""HMMT Feb 2026. No official standalone release; MathArena community curation.""" from ..sample import Sample from ..registry import register_dataset @@ -8,12 +8,12 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='hmmt26', - source='evalscope/hmmt_feb_2026', # curated; no official upstream release (ModelScope) - split='train', # the mirror ships a single train split + source='MathArena/hmmt_feb_2026', # curated by MathArena (HuggingFace) + split='train', # the dataset ships a single split task_type='math', tags=['math', 'competition'], description='HMMT February 2026 (community-curated, no official upstream).', - params={'hub': 'modelscope'}, + params={'hub': 'hf_raw'}, ) ) def hmmt26(): diff --git a/evalharness/data/datasets/imo_answerbench.py b/evalharness/data/datasets/imo_answerbench.py index b01e50a..cc919cf 100644 --- a/evalharness/data/datasets/imo_answerbench.py +++ b/evalharness/data/datasets/imo_answerbench.py @@ -8,12 +8,12 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='imo_answerbench', - source='evalscope/imo-answerbench', # curated; no official upstream release (ModelScope) - split='train', # the mirror ships a single train split + source='OpenEvals/IMO-AnswerBench', # HF OpenEvals mirror of the community curation + split='train', # the dataset ships a single split task_type='math', tags=['math', 'competition', 'imo'], description='IMO-level answer bench (community-curated, no official upstream).', - params={'hub': 'modelscope'}, + params={'hub': 'hf_raw'}, ) ) def imo_answerbench(): diff --git a/evalharness/data/datasets/live_code_bench.py b/evalharness/data/datasets/live_code_bench.py index 11ab3b9..cb8852b 100644 --- a/evalharness/data/datasets/live_code_bench.py +++ b/evalharness/data/datasets/live_code_bench.py @@ -8,14 +8,14 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='live_code_bench', - source='evalscope/livecodebench_code_generation_lite_parquet', # parquet mirror (ModelScope); - # the HF original is script-based and no longer loadable by datasets>=5 - subset='release_latest', # or release_v1..v6 / vX / vX_vY version windows + source='sam-paech/livecodebench-code_generation_lite', # HF parquet conversion; the official + # livecodebench/code_generation_lite is script-based (unloadable by datasets>=5) + subset='release_latest', # or release_v1..v6 split='test', task_type='coding', tags=['code'], description='LiveCodeBench (lite) contest problems; version tags via DatasetSpec.version.', - params={'hub': 'modelscope'}, + params={'hub': 'hf_raw'}, ) ) def live_code_bench(): diff --git a/evalharness/data/datasets/openai_mrcr.py b/evalharness/data/datasets/openai_mrcr.py index 1be7ba8..1ae41e9 100644 --- a/evalharness/data/datasets/openai_mrcr.py +++ b/evalharness/data/datasets/openai_mrcr.py @@ -8,13 +8,13 @@ from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='openai_mrcr', - source='openai-mirror/mrcr', # OpenAI MRCR mirror on ModelScope + source='openai/mrcr', # OFFICIAL OpenAI release (HuggingFace) subset='2needle', # or 4needle / 8needle split='test', task_type='qa', tags=['long_context'], - description='OpenAI MRCR long-context retrieval/coreference (community mirror).', - params={'hub': 'modelscope'}, + description='OpenAI MRCR long-context retrieval/coreference (official).', + params={'hub': 'hf_raw'}, ) ) def openai_mrcr(): diff --git a/evalharness/data/datasets/tau2_bench.py b/evalharness/data/datasets/tau2_bench.py index 5e94084..384f329 100644 --- a/evalharness/data/datasets/tau2_bench.py +++ b/evalharness/data/datasets/tau2_bench.py @@ -15,13 +15,13 @@ _DOMAINS = ('airline', 'retail', 'telecom', 'mock') @register_dataset( DatasetSpec( name='tau2_bench', - source='evalscope/tau2-bench-data', # mirror of the official GitHub data + source='HuggingFaceH4/tau2-bench-data', # HF mirror of the official GitHub data subset='airline', # or retail / telecom / mock; override with --subset split='test', task_type='agent', tags=['agent', 'tool_use', 'dialog'], - description='tau2-bench agent-tool-dialog tasks (official content, ModelScope mirror).', - params={'hub': 'modelscope', 'ms_files': ['tau2/domains/{subset}/tasks.json']}, + description='tau2-bench agent-tool-dialog tasks (official content, HF mirror).', + params={'hub': 'hf_raw', 'hf_files': ['domains/{subset}/tasks.json']}, ) ) def tau2_bench(): diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py index e650bef..9b03a0d 100644 --- a/evalharness/data/loader.py +++ b/evalharness/data/loader.py @@ -24,7 +24,7 @@ from .spec import DatasetSpec, FieldSpec _SUPPORTED_EXTS = ['.jsonl', '.json', '.csv', '.tsv', '.parquet'] # params keys consumed by the loader itself; never forwarded to load_dataset() -_RESERVED_PARAMS = {'hub', 'ms_files', 'filter_column'} +_RESERVED_PARAMS = {'hub', 'ms_files', 'hf_files', 'filter_column'} _MS_API = 'https://www.modelscope.cn/api/v1/datasets' @@ -52,6 +52,8 @@ def load_raw_records(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[ _link_or_copy_all(native_files, raw_dir) elif spec.params.get('hub') == 'modelscope': records = _load_from_modelscope(spec, raw_dir) + elif spec.params.get('hub') == 'hf_raw': + records = _load_from_hf_raw(spec, raw_dir) else: records = _load_from_hub(spec) if raw_dir is not None: @@ -241,6 +243,127 @@ def _load_from_modelscope(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> return records +# ---------------- HuggingFace raw-file loading (no auth, exact bytes) ---------------- +# For repos whose layout datasets.load_dataset cannot express (raw json trees, +# shard dirs without configs) or that only exist as plain files. Honors +# $HF_ENDPOINT (e.g. https://hf-mirror.com). + + +def _hf_base() -> str: + return os.environ.get('HF_ENDPOINT', 'https://huggingface.co').rstrip('/') + + +def _hf_list_files(repo: str, root: str = '', depth: int = 0) -> List[str]: + """Recursively list file paths under an HF dataset repo directory.""" + url = f'{_hf_base()}/api/datasets/{repo}/tree/main' + (f'/{root}' if root else '') + '?limit=1000' + req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'}) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + entries = json.loads(resp.read().decode('utf-8')) + except Exception: + return [] + files: List[str] = [] + for entry in entries: + if entry.get('type') == 'file': + files.append(entry['path']) + elif entry.get('type') == 'directory' and depth < 4: + files.extend(_hf_list_files(repo, entry['path'], depth + 1)) + return files + + +def _hf_match_files(spec: DatasetSpec, files: List[str]) -> List[str]: + """Pick repo files for this subset/split (see module docstring conventions).""" + explicit = spec.params.get('hf_files') + if explicit: + return [p.format(subset=spec.subset, split=spec.split) for p in explicit] + + def ext_ok(p: str) -> bool: + return os.path.splitext(p)[1] in _SUPPORTED_EXTS + + data = [f for f in files if ext_ok(f)] + if spec.subset != 'default': + # subset wins exclusively: never ALSO match bare-split shards in the + # same dir (repos like sam-paech LCB mix test-*.parquet and + # release_v*-*.parquet in one data/ folder) + in_dir = [f for f in data if os.path.dirname(f) == spec.subset] + if in_dir: + return sorted(in_dir) + exact = [] + for f in data: + stem = os.path.splitext(os.path.basename(f))[0] + if stem in (f'{spec.subset}_{spec.split}', spec.subset): + exact.append(f) + if exact: + return sorted(exact) + shards = [ + f for f in data + if re.fullmatch(rf'{re.escape(spec.subset)}[-_]\d+(-of-\d+)?' + rf'|{re.escape(spec.subset)}_{re.escape(spec.split)}[-_].*', + os.path.splitext(os.path.basename(f))[0]) + ] + if shards: + return sorted(shards) + else: + exact = [ + f for f in data + if os.path.splitext(os.path.basename(f))[0] == spec.split + ] + if exact: + return sorted(exact) + shards = [ + f for f in data + if re.fullmatch(rf'{re.escape(spec.split)}[-_]\d+(-of-\d+)?', + os.path.splitext(os.path.basename(f))[0]) + ] + if shards: + return sorted(shards) + if len(data) == 1: + return data + return [] + + +def _hf_download(repo: str, path: str, dest_dir: Path) -> Path: + """Download one repo file (follows the CDN redirect) into the blob store.""" + dest = dest_dir / os.path.basename(path) + if dest.exists() and dest.stat().st_size > 0: + return dest + dest_dir.mkdir(parents=True, exist_ok=True) + url = f'{_hf_base()}/datasets/{repo}/resolve/main/{path}' + tmp = dest.with_name(dest.name + f'.part-{os.getpid()}') + req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'}) + with urllib.request.urlopen(req, timeout=1800) as resp, open(tmp, 'wb') as out: + while True: + chunk = resp.read(1 << 20) + if not chunk: + break + out.write(chunk) + os.replace(tmp, dest) + return dest + + +def _load_from_hf_raw(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[Dict[str, Any]]: + import hashlib + + files = _hf_list_files(spec.source) + if not files: + raise FileNotFoundError(f'no files found on HF dataset {spec.source!r}') + selected = _hf_match_files(spec, files) + if not selected: + raise FileNotFoundError( + f'no data file for subset={spec.subset!r} split={spec.split!r} in ' + f'HF {spec.source!r}. Available (first 10): {files[:10]}' + ) + blob_dir = get_cache_root() / '.raw' / hashlib.md5(spec.source.encode()).hexdigest()[:10] + records: List[Dict[str, Any]] = [] + blobs: List[Path] = [] + for path in selected: + blobs.append(_hf_download(spec.source, path, blob_dir)) + records.extend(_read_file(str(blobs[-1]))) + if raw_dir is not None: + _link_or_copy_all(blobs, raw_dir) + return records + + def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]: try: import datasets diff --git a/evalharness/eval/__init__.py b/evalharness/eval/__init__.py new file mode 100644 index 0000000..a10108c --- /dev/null +++ b/evalharness/eval/__init__.py @@ -0,0 +1,47 @@ +"""evalharness.eval -- the evaluation layer. + +Pipeline: extract -> score -> aggregate, all plugin-driven. + + from evalharness.eval import evaluate, get_eval + from evalharness import get_dataset + + ds = get_dataset('gsm8k') + report = evaluate(ds, predictions) # recipe auto-resolved by dataset name + report.save('gsm8k.report.json') + +Four scoring paradigms: text-compare (implemented), llm-judge (wired via +runner judge= once ModelAdapter exists), execution & env-reward (slots raise +LayerNotReady until sandbox/agent layers land). +""" + +from .aggregator import AGGREGATOR_REGISTRY, get_aggregator, register_aggregator +from .extractor import EXTRACTOR_REGISTRY, get_extractor, make_extractor, register_extractor +from .recipe import EVAL_REGISTRY, EvalRecipe, JudgeConfig, get_eval, list_evals, register_eval +from .record import EvalReport, SampleResult +from .registry import EvalRegistry +from .runner import evaluate +from .scorer import SCORER_REGISTRY, LayerNotReady, ScoreContext, get_scorer, register_scorer + +__all__ = [ + 'evaluate', 'EvalRecipe', 'JudgeConfig', 'get_eval', 'list_evals', 'register_eval', + 'EvalReport', 'SampleResult', 'LayerNotReady', 'ScoreContext', + 'EXTRACTOR_REGISTRY', 'SCORER_REGISTRY', 'AGGREGATOR_REGISTRY', 'EvalRegistry', + 'register_extractor', 'get_extractor', 'make_extractor', + 'register_scorer', 'get_scorer', 'register_aggregator', 'get_aggregator', +] + + +def _discover_builtin_recipes() -> None: + """Import every recipe module under ./recipes (import = register).""" + import importlib + import pkgutil + from pathlib import Path + + pkg_dir = Path(__file__).parent / 'recipes' + if not pkg_dir.exists(): + return + for info in pkgutil.iter_modules([str(pkg_dir)]): + importlib.import_module(f'{__name__}.recipes.{info.name}') + + +_discover_builtin_recipes() diff --git a/evalharness/eval/aggregator.py b/evalharness/eval/aggregator.py new file mode 100644 index 0000000..1aac8e2 --- /dev/null +++ b/evalharness/eval/aggregator.py @@ -0,0 +1,124 @@ +"""Aggregator primitives: fold per-sample scores into report metrics. + +Aggregation is NOT always mean: pass@k groups by task first, MRCR averages +inside length bins, BFCL averages per category then (optionally weights). + +Contract: fn(results: List[SampleResult], metric: str) -> AggOut + AggOut = float | dict[str, float] (dict -> nested metric_groups) +Register: @register_aggregator('mean') +""" + +import math +from collections import defaultdict +from typing import Callable, Dict, List, Union + +from .record import SampleResult +from .registry import EvalRegistry + +AggregatorFn = Callable[[List[SampleResult], str], Union[float, Dict[str, float]]] + +AGGREGATOR_REGISTRY = EvalRegistry('aggregator') + + +def register_aggregator(name: str): + def decorator(fn: AggregatorFn) -> AggregatorFn: + AGGREGATOR_REGISTRY.register(name, fn) + return fn + + return decorator + + +def get_aggregator(name: str) -> AggregatorFn: + return AGGREGATOR_REGISTRY.get(name) + + +@register_aggregator('mean') +def mean(results: List[SampleResult], metric: str): + vals = [r.scores.get(metric, 0.0) for r in results if metric in r.scores] + return sum(vals) / len(vals) if vals else 0.0 + + +@register_aggregator('pass_at_k') +def pass_at_k(results: List[SampleResult], metric: str): + """Unbiased pass@k over per-task sample groups (HumaneEval convention). + + Uses group_key = task id; each group holds n samples with c passes. + Reports pass@1..min(k_max, max group size). params read from metric name + suffix is NOT used -- k list comes from the recipe binding. + The recipe binds: aggregator={'pass@k': ('pass_at_k', {'k': [1, 2, 8]})} + which the runner expands into per-k metric names before calling. + """ + groups: Dict[str, List[float]] = defaultdict(list) + for r in results: + if metric in r.scores: + groups[r.group_key or str(r.sample_id)].append(r.scores[metric]) + if not groups: + return 0.0 + total = 0.0 + for runs in groups.values(): + total += sum(runs) / len(runs) # per-task pass rate; unbiased for k=1 + return total / len(groups) + + +def unbiased_pass_at_k(n: int, c: int, k: int) -> float: + """1 - C(n-c, k) / C(n, k) -- the official HumanEval estimator.""" + if n - c < k: + return 1.0 + return 1.0 - math.prod(1.0 - k / i for i in range(n - c + 1, n + 1)) + + +@register_aggregator('grouped_avg') +def grouped_avg(results: List[SampleResult], metric: str): + """Average within each group_key, return {group: avg} (BFCL categories).""" + buckets: Dict[str, List[float]] = defaultdict(list) + for r in results: + if metric in r.scores: + buckets[r.group_key or 'default'].append(r.scores[metric]) + return {g: sum(v) / len(v) for g, v in sorted(buckets.items())} + + +@register_aggregator('weighted_group_avg') +def weighted_group_avg(results: List[SampleResult], metric: str): + """Group averages + a sample-weighted overall (BFCL unweighted vs weighted).""" + buckets: Dict[str, List[float]] = defaultdict(list) + for r in results: + if metric in r.scores: + buckets[r.group_key or 'default'].append(r.scores[metric]) + out = {f'{g}': sum(v) / len(v) for g, v in sorted(buckets.items())} + all_vals = [x for v in buckets.values() for x in v] + out['overall'] = sum(all_vals) / len(all_vals) if all_vals else 0.0 + return out + + +@register_aggregator('simpleqa_official') +def simpleqa_official(results: List[SampleResult], metric: str): + """Official SimpleQA aggregate: rates + is_given_attempted + accuracy_given_attempted. + + Returns flat metrics under derived_ keys; the runner folds dicts into + metric_groups, so we emit {name: value} for the report. + """ + n = sum(1 for r in results if metric in r.scores) + if not n: + return 0.0 + correct = sum(r.scores[metric] for r in results if metric in r.scores) + incorrect = sum(r.scores.get('is_incorrect', 0.0) for r in results) + not_attempted = sum(r.scores.get('is_not_attempted', 0.0) for r in results) + attempted = incorrect + correct + return { + 'is_correct': correct / n, + 'is_incorrect': incorrect / n, + 'is_not_attempted': not_attempted / n, + 'is_given_attempted': attempted / n, + 'accuracy_given_attempted': (correct / attempted) if attempted > 0 else 0.0, + } + + +@register_aggregator('binned_avg') +def binned_avg(results: List[SampleResult], metric: str): + """Average inside metadata['bin'] buckets (MRCR length bins).""" + buckets: Dict[str, List[float]] = defaultdict(list) + for r in results: + if metric in r.scores: + b = str(r.metadata.get('bin', r.group_key or 'default')) + buckets[b].append(r.scores[metric]) + return {b: sum(v) / len(v) for b, v in sorted(buckets.items(), key=lambda kv: kv[0])} diff --git a/evalharness/eval/extractor.py b/evalharness/eval/extractor.py new file mode 100644 index 0000000..0672173 --- /dev/null +++ b/evalharness/eval/extractor.py @@ -0,0 +1,212 @@ +"""Extractor primitives: pull a comparable answer string out of a raw prediction. + +Extractors are shared, reusable building blocks -- NOT per-bench copies. +A recipe selects primitives (by name, or a custom fn) and may cascade them; +the most specific pattern goes first, the fallback last. + +Contract: fn(raw_prediction: str, sample: Sample) -> (str, ok: bool, note: str) +Register: @register_extractor('math_boxed') +Look up: get_extractor('math_boxed') / make_extractor({'cascade': [...]} or 'name' or fn) +""" + +import re +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from ..data.sample import Sample +from .registry import EvalRegistry + +ExtractorFn = Callable[[str, Sample], Tuple[str, bool, str]] + +EXTRACTOR_REGISTRY = EvalRegistry('extractor') + + +def register_extractor(name: str): + def decorator(fn: ExtractorFn) -> ExtractorFn: + EXTRACTOR_REGISTRY.register(name, fn) + return fn + + return decorator + + +def get_extractor(name: str) -> ExtractorFn: + return EXTRACTOR_REGISTRY.get(name) + + +ExtractorSpec = Union[str, ExtractorFn, List[Union[str, ExtractorFn]], None] + + +def make_extractor(spec: ExtractorSpec) -> ExtractorFn: + """Resolve a recipe's extract spec into one callable. + + - 'name' -> registered primitive + - fn -> custom function (already the right signature) + - ['a', 'b', fn] -> cascade: first stage that succeeds wins + - None -> identity (whole prediction, minus whitespace) + """ + if spec is None: + return identity + if callable(spec): + return spec + if isinstance(spec, str): + return get_extractor(spec) + if isinstance(spec, list): + stages = [make_extractor(s) for s in spec] + if not stages: + raise ValueError('empty extractor cascade') + + def cascade(raw: str, sample: Sample): + last_note = 'all stages empty' + for fn in stages: + value, ok, note = fn(raw, sample) + if ok: + return value, True, note + last_note = note + return '', False, last_note + + return cascade + raise TypeError(f'bad extractor spec: {spec!r}') + + +# ------------------------- primitives ------------------------- + + +@register_extractor('identity') +def identity(raw: str, sample: Sample) -> Tuple[str, bool, str]: + text = (raw or '').strip() + return text, bool(text), 'identity' + + +@register_extractor('math_boxed') +def math_boxed(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """Last \\boxed{...} with brace balancing (Qwen/Hendrycks convention).""" + text = raw or '' + idx = text.rfind('\\boxed{') + if idx < 0: + return '', False, 'no boxed' + i = idx + len('\\boxed{') + depth, out = 1, [] + while i < len(text) and depth: + if text[i] == '{': + depth += 1 + out.append('{') + elif text[i] == '}': + depth -= 1 + if depth == 0: + break + out.append('}') + else: + out.append(text[i]) + i += 1 + if depth != 0: + return '', False, 'unbalanced boxed' + value = ''.join(out).strip() + return value, bool(value), 'boxed' + + +_NUMBER_TAIL = re.compile(r'-?\d[\d,]*\.?\d*') +_ANSWER_IS = re.compile( + r'(?:the answer is|final answer is|answer:|ANSWER:|答案是)\s*:?\s*(.+)', re.IGNORECASE) +_ANSWER_DOLLAR = re.compile(r'final answer is \$([^$]+)\$') + + +@register_extractor('answer_phrase') +def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """Text after the last 'the answer is' / 'ANSWER:' / '答案是' marker.""" + m = None + for m in _ANSWER_IS.finditer(raw or ''): + pass + if not m: + return '', False, 'no answer phrase' + value = m.group(1).strip().strip('$.: ').split('\n')[0].strip() + return value, bool(value), f'phrase:{m.group(0)[:20].strip()}' + + +@register_extractor('last_number') +def last_number(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """Final number in the text (gsm8k/AIME fallback).""" + nums = _NUMBER_TAIL.findall((raw or '').replace(',', '')) + if not nums: + return '', False, 'no number' + value = nums[-1].rstrip('.') + return value, bool(value), 'last_number' + + +@register_extractor('gsm8k_hash') +def gsm8k_hash(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """`#### 42` marker (gsm8k few-shot convention).""" + m = re.findall(r'####\s*(-?[\d.,]+)', raw or '') + if not m: + return '', False, 'no ####' + return m[-1].replace(',', '').strip('.'), True, 'gsm8k_hash' + + +_LETTER_PAREN = re.compile(r'\(([A-J])\)', re.IGNORECASE) +_LETTER_BARE = re.compile(r'\b([A-J])\b') +_LETTER_CN = re.compile(r'答案是\s*\(?([A-J])\)?', re.IGNORECASE) + + +@register_extractor('mcq_letter') +def mcq_letter(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """Multiple-choice letter: prefer (A) style, then 答案是X, then bare A.""" + text = raw or '' + tail = text[text.rfind('answer'):] if 'answer' in text.lower() else text + m = None + for m in _LETTER_PAREN.finditer(tail): + pass + if m: + return m.group(1).upper(), True, 'letter_paren' + m = _LETTER_CN.search(text) + if m: + return m.group(1).upper(), True, 'letter_cn' + for m in _LETTER_BARE.finditer(tail): + pass + if m: + return m.group(1).upper(), True, 'letter_bare' + return '', False, 'no letter' + + +_CODE_BLOCK = re.compile(r'```(?:[a-zA-Z0-9_+-]*)\s*\n(.*?)```', re.DOTALL) + + +@register_extractor('code_block') +def code_block(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """First (or all, joined) fenced code block; falls back to whole text.""" + blocks = _CODE_BLOCK.findall(raw or '') + if blocks: + return blocks[0].strip('\n'), True, 'code_block' + stripped = (raw or '').strip() + if stripped.startswith(('def ', 'class ', 'import ', 'from ')): + return stripped, True, 'whole_is_code' + return '', False, 'no code block' + + +@register_extractor('quoted_list') +def quoted_list(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """MRCR style: the model repeats markers as QUOTED strings.""" + quotes = re.findall(r'"([^"\n]{2,})"', raw or '') + if not quotes: + return '', False, 'no quotes' + return '\n'.join(quotes), True, 'quoted_list' + + +@register_extractor('answer_spans') +def answer_spans(raw: str, sample: Sample) -> Tuple[str, bool, str]: + """DROP multi-span: collect EVERY `Answer:` line, newline-joined. + + Official pattern captures one line per match ([^\\n]+); multiple Answer: + lines (or repeated answers) each contribute one span, matching the gold + spans-tuple format. + """ + matches = re.findall(r'(?i)Answer\s*:\s*([^\n]+)', raw or '') + if not matches: + return '', False, 'no Answer: line' + spans = [m.strip() for m in matches if m.strip()] + if not spans: + return '', False, 'empty Answer:' + return '\n'.join(spans), True, f'answer_spans:{len(spans)}' + + +@register_extractor('first_line') +def first_line(raw: str, sample: Sample) -> Tuple[str, bool, str]: + line = (raw or '').strip().split('\n')[0].strip() + return line, bool(line), 'first_line' diff --git a/evalharness/eval/math_grader.py b/evalharness/eval/math_grader.py new file mode 100644 index 0000000..2074d9e --- /dev/null +++ b/evalharness/eval/math_grader.py @@ -0,0 +1,145 @@ +"""Optional sympy-based math grader (adapted from OpenAI PRM800K, MIT license). + +Imported lazily by the math_equal scorer; only needed when normalized string +equality is not enough (unreduced fractions, symbolic forms, units). +Requires: pip install sympy pylatexenc +""" + +import logging +from typing import Optional + +logger = logging.getLogger(__name__) + +try: + import sympy + from pylatexenc import latex2text + from sympy.parsing import sympy_parser + + _OK = True +except ImportError: # pragma: no cover - optional dependency + _OK = False + +BAD_SUBSTRINGS = ('^{', '^(') +TUPLE_CHARS = '()[]' + + +def _sympy_parse(expr: str): + return sympy_parser.parse_expr( + expr.replace('^', '**'), + transformations=( + sympy_parser.standard_transformations + + (sympy_parser.implicit_multiplication_application,) + ), + ) + + +def _parse_latex(expr: str) -> str: + expr = expr.replace('\\tfrac', '\\frac').replace('\\dfrac', '\\frac') + expr = latex2text.LatexNodes2Text().latex_to_text(expr) + return (expr.replace('√', 'sqrt').replace('π', 'pi').replace('·', '*') + .replace('×', '*').strip()) + + +def _strip_commas(expr: str) -> str: + import re + + return re.sub(r'(\d),(\d\d\d)(?=\D|$)', r'\1\2', expr) + + +def _is_float(x) -> bool: + try: + float(x) + return True + except (TypeError, ValueError): + return False + + +def _str_is_int(x: str) -> bool: + try: + return abs(float(x) - int(round(float(x)))) <= 1e-7 + except (TypeError, ValueError): + return False + + +def _is_frac(expr: str) -> bool: + import re + + return bool(re.fullmatch(r'-?[0-9]+.?/0*[1-9][0-9]*.?', expr or '')) + + +def _normalize(expr: Optional[str]) -> Optional[str]: + import re + + if expr is None: + return None + m = re.fullmatch(r'\\text\{(.+?)\}', expr) + if m: + expr = m.group(1) + expr = (expr.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '') + .replace(' or ', ' , ').replace(' and ', ' , ') + .replace('million', '*10^6').replace('billion', '*10^9')) + for unit in ('degree', 'cm', 'meter', 'mile', 'second', 'minute', 'hour', + 'day', 'week', 'month', 'year', 'foot', 'feet', 'inch', 'yard'): + expr = re.sub(unit + r'(es)?(s)? *(\^[0-9]+)?', '', expr) + expr = re.sub(r'\^ *\\circ', '', expr) + if len(expr) > 1 and expr[0] == '{' and expr[-1] == '}': + expr = expr[1:-1] + expr = _strip_commas(expr) + if _is_float(expr): + try: + f = float(expr) + if abs(f - round(f)) <= 1e-7: + expr = str(int(round(f))) + except (TypeError, ValueError): + pass + if '\\' in expr: + try: + expr = _parse_latex(expr) + except Exception: + pass + expr = re.sub(r'- *', '-', expr.replace(' ', '')) + expr = expr.replace('{', '').replace('}', '').lower() + if _str_is_int(expr): + expr = str(int(round(float(expr)))) + return expr + + +def _split_tuple(expr: str): + expr = _strip_commas(expr) + if (len(expr) > 2 and expr[0] in TUPLE_CHARS and expr[-1] in TUPLE_CHARS + and all(c not in expr[1:-1] for c in TUPLE_CHARS)): + return [e.strip() for e in expr[1:-1].split(',')] + return [expr] + + +def _sympy_equal(a: str, b: str) -> bool: + try: + diff = _sympy_parse(f'({a})-({b})') + return sympy.simplify(diff) == 0 + except Exception: + return False + + +def grade_answer(given_answer: str, ground_truth: str) -> bool: + """True iff equal under normalization or sympy simplification.""" + if not _OK: + raise ImportError('pip install sympy pylatexenc for symbolic math grading') + if given_answer is None: + return False + a, b = _normalize(given_answer), _normalize(ground_truth) + if a == b: + return True + if not a or not b: + return False + a_elems, b_elems = _split_tuple(a), _split_tuple(b) + if len(a_elems) != len(b_elems): + return False + for x, y in zip(a_elems, b_elems): + if _is_frac(x) and _is_frac(y): + if x != y: + return False + elif _str_is_int(x) != _str_is_int(y): + return False + elif not _sympy_equal(x, y): + return False + return True diff --git a/evalharness/eval/recipe.py b/evalharness/eval/recipe.py new file mode 100644 index 0000000..046b53f --- /dev/null +++ b/evalharness/eval/recipe.py @@ -0,0 +1,82 @@ +"""EvalRecipe: what a benchmark's evaluation IS, declared not coded. + +Mirrors the data-layer plugin shape: a recipe binds + extract -- extractor spec (primitive name / cascade / custom fn) + scorers -- {metric: scorer spec}; spec = name | fn | {'name': ..., **params} + aggregators -- {metric: aggregator name | (name, params)} (default 'mean') + judge -- optional LLM-judge config (model tag; wiring comes later) + +Registry: @register_eval('gsm8k') -> get_eval('gsm8k') -> EvalRecipe +""" + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from .aggregator import get_aggregator +from .extractor import ExtractorSpec, make_extractor +from .scorer import ScorerSpec, make_scorer +from .registry import EvalRegistry + +EVAL_REGISTRY = EvalRegistry('eval recipe') + + +def register_eval(name: str): + def decorator(factory: Callable[[], 'EvalRecipe']): + EVAL_REGISTRY.register(name, factory) + return factory + + return decorator + + +def get_eval(name: str) -> 'EvalRecipe': + return EVAL_REGISTRY.get(name)() + + +def list_evals() -> List[str]: + return EVAL_REGISTRY.names() + + +@dataclass +class JudgeConfig: + """LLM-judge wiring; the actual callable is injected by the runner.""" + + model: str = '' # model tag / url, resolved by ModelAdapter later + temperature: float = 0.0 + max_retries: int = 2 + + +@dataclass +class EvalRecipe: + name: str = '' + extract: ExtractorSpec = None + scorers: Dict[str, ScorerSpec] = field(default_factory=dict) + # metric -> aggregator name | (name, params); missing -> 'mean' + aggregators: Dict[str, Union[str, Tuple[str, Dict[str, Any]]]] = field(default_factory=dict) + judge: Optional[JudgeConfig] = None + description: str = '' + + def resolve_extract(self): + return make_extractor(self.extract) + + def resolve_scorers(self) -> Dict[str, Callable]: + if not self.scorers: + raise ValueError(f'recipe {self.name!r} has no scorers') + return {m: make_scorer(m, s) for m, s in self.scorers.items()} + + def resolve_aggregators(self) -> Dict[str, Callable]: + out = {} + for metric, spec in self.aggregators.items(): + if isinstance(spec, tuple): + name, params = spec + base = get_aggregator(name) + + def with_params(results, m, _base=base, _params=params): + return _base(results, m, **_params) if _params else _base(results, m) + + out[metric] = with_params + else: + out[metric] = get_aggregator(spec or 'mean') + return out + + def primary_metric(self) -> str: + return next(iter(self.scorers), 'acc') diff --git a/evalharness/eval/recipes/__init__.py b/evalharness/eval/recipes/__init__.py new file mode 100644 index 0000000..7cd9d00 --- /dev/null +++ b/evalharness/eval/recipes/__init__.py @@ -0,0 +1,5 @@ +"""Built-in eval recipes, one module per benchmark family. + +Each recipe binds shared extractor/scorer/aggregator primitives; custom +per-bench logic lives here ONLY when no primitive fits. +""" diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py new file mode 100644 index 0000000..418a331 --- /dev/null +++ b/evalharness/eval/recipes/agent.py @@ -0,0 +1,78 @@ +"""Execution / agent benchmarks. Recipes exist now; scorers raise LayerNotReady +until the sandbox & agent layers land (interfaces are stable).""" + +from ..recipe import EvalRecipe, register_eval + + +@register_eval('humaneval') +def humaneval(): + return EvalRecipe( + name='humaneval', + extract='code_block', + scorers={'pass@1': 'execution'}, + aggregators={'pass@1': 'pass_at_k'}, + description='HumanEval; sandbox test execution, pass@k.', + ) + + +@register_eval('bigcodebench') +def bigcodebench(): + return EvalRecipe( + name='bigcodebench', + extract='code_block', + scorers={'pass@1': 'execution'}, + aggregators={'pass@1': 'pass_at_k'}, + description='BigCodeBench; sandbox test execution with libs, pass@k.', + ) + + +@register_eval('live_code_bench') +def live_code_bench(): + return EvalRecipe( + name='live_code_bench', + extract='code_block', + scorers={'pass@1': 'execution'}, + aggregators={'pass@1': 'pass_at_k'}, + description='LiveCodeBench; hidden tests, pass@k.', + ) + + +@register_eval('swe_bench_verified') +def swe_bench_verified(): + return EvalRecipe( + name='swe_bench_verified', + extract='identity', # a patch, not an answer + scorers={'resolved': 'env_reward'}, + description='SWE-bench Verified; docker env, FAIL_TO_PASS/PASS_TO_PASS.', + ) + + +@register_eval('tau2_bench') +def tau2_bench(): + return EvalRecipe( + name='tau2_bench', + extract='identity', + scorers={'acc': 'env_reward'}, + description='tau2-bench; user-simulated dialog, environment reward.', + ) + + +@register_eval('bfcl_v3') +def bfcl_v3(): + return EvalRecipe( + name='bfcl_v3', + extract='identity', + scorers={'acc': 'execution'}, # AST check for most categories; exec for executable ones + aggregators={'acc': 'weighted_group_avg'}, # group_key = test_category + description='BFCL v3; AST/exec per category, weighted category average.', + ) + + +@register_eval('general_fc') +def general_fc(): + return EvalRecipe( + name='general_fc', + extract='identity', + scorers={'acc': 'execution'}, + description='General function calling; tool-call comparison.', + ) diff --git a/evalharness/eval/recipes/judged.py b/evalharness/eval/recipes/judged.py new file mode 100644 index 0000000..b3e0806 --- /dev/null +++ b/evalharness/eval/recipes/judged.py @@ -0,0 +1,60 @@ +"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter.""" + +from ..recipe import EvalRecipe, JudgeConfig, register_eval + +_HLE_PROMPT = ( + 'Judge whether the following [response] to [question] is correct or not based ' + 'on the precise and unambiguous [correct_answer] below.\n\n' + '[question]: {question}\n\n[response]: {prediction}\n\n' + '[correct_answer]: {target}\n\n' + 'Focus only on whether the answers match. In one or two sentences explain, then ' + "write your final line as 'GRADE: C' for correct or 'GRADE: I' for incorrect." +) + +_SIMPLE_QA_PROMPT = ( + 'Your job is to look at a question, a gold target, and a predicted answer, and then ' + 'assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"].\n' + 'Judge by: a predicted answer is CORRECT iff it fully contains the important ' + 'information in the gold target and contains no contradicting information; hedged but ' + 'complete answers are CORRECT; answers missing the key information without ' + 'contradiction are NOT_ATTEMPTED. For numeric gold targets the prediction must be ' + 'correct to the last significant figure of the gold answer.\n\n' + 'Question: {question}\nGold target: {target}\nPredicted answer: {prediction}\n\n' + 'Grade as one of:\nA: CORRECT\nB: INCORRECT\nC: NOT_ATTEMPTED\n\n' + 'Just return the letter "A", "B", or "C" with no text around it.' +) + + +@register_eval('hle') +def hle(): + return EvalRecipe( + name='hle', + extract='identity', + scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT, + 'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}}, + judge=JudgeConfig(model='judge'), + description="HLE; official GRADE: C/I LLM judge.", + ) + + +@register_eval('simple_qa') +def simple_qa(): + return EvalRecipe( + name='simple_qa', + extract='identity', + scorers={ + 'is_correct': { + 'name': 'llm_judge', + 'prompt_template': _SIMPLE_QA_PROMPT, + # official grading: match A|B|C, default to C (NOT_ATTEMPTED) + 'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0}, + 'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0}, + 'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}}, + 'default_label': 'C', + 'primary': 'is_correct', + }, + }, + aggregators={'is_correct': 'simpleqa_official'}, + judge=JudgeConfig(model='judge'), + description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.', + ) diff --git a/evalharness/eval/recipes/math.py b/evalharness/eval/recipes/math.py new file mode 100644 index 0000000..c046e3a --- /dev/null +++ b/evalharness/eval/recipes/math.py @@ -0,0 +1,50 @@ +"""Math benchmarks: gsm8k, aime24/25/26, hmmt26, imo_answerbench, competition_math.""" + +from ..recipe import EvalRecipe, register_eval + +MATH_EXTRACT = ['math_boxed', 'answer_phrase', 'last_number'] +MATH_SCORE = {'acc': 'math_equal'} + + +@register_eval('gsm8k') +def gsm8k(): + return EvalRecipe( + name='gsm8k', + extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'], + scorers={'acc': {'name': 'math_equal', 'sympy': False}}, # integer answers: no sympy needed + description='Grade-school math; #### and boxed markers, numeric compare.', + ) + + +def _math_comp(name: str, desc: str) -> EvalRecipe: + return EvalRecipe(name=name, extract=MATH_EXTRACT, scorers=MATH_SCORE, description=desc) + + +@register_eval('aime24') +def aime24(): + return _math_comp('aime24', 'AIME 2024; boxed extraction + sympy equivalence.') + + +@register_eval('aime25') +def aime25(): + return _math_comp('aime25', 'AIME 2025; boxed extraction + sympy equivalence.') + + +@register_eval('aime26') +def aime26(): + return _math_comp('aime26', 'AIME 2026; boxed extraction + sympy equivalence.') + + +@register_eval('hmmt26') +def hmmt26(): + return _math_comp('hmmt26', 'HMMT Feb 2026; boxed extraction + sympy equivalence.') + + +@register_eval('imo_answerbench') +def imo_answerbench(): + return _math_comp('imo_answerbench', 'IMO AnswerBench; boxed extraction + sympy equivalence.') + + +@register_eval('competition_math') +def competition_math(): + return _math_comp('competition_math', 'Hendrycks MATH; boxed + sympy (PRM800K grader).') diff --git a/evalharness/eval/recipes/mcq.py b/evalharness/eval/recipes/mcq.py new file mode 100644 index 0000000..51e4b29 --- /dev/null +++ b/evalharness/eval/recipes/mcq.py @@ -0,0 +1,53 @@ +"""MCQ benchmarks: mmlu, cmmlu, mmlu_pro, arc, hellaswag, winogrande, +gpqa_diamond, longbench_v2. All: extract a letter, compare to target letter.""" + +from ..recipe import EvalRecipe, register_eval + + +def _mcq(name: str, desc: str) -> EvalRecipe: + return EvalRecipe( + name=name, + extract=['mcq_letter'], + scorers={'acc': {'name': 'exact', 'mode': 'raw'}}, # letter == letter + description=desc, + ) + + +@register_eval('mmlu') +def mmlu(): + return _mcq('mmlu', 'MMLU; letter extraction vs answer letter.') + + +@register_eval('cmmlu') +def cmmlu(): + return _mcq('cmmlu', 'CMMLU; letter extraction (supports 答案是X) vs answer letter.') + + +@register_eval('mmlu_pro') +def mmlu_pro(): + return _mcq('mmlu_pro', 'MMLU-Pro 10-option; letter vs letter.') + + +@register_eval('arc') +def arc(): + return _mcq('arc', 'AI2 ARC; letter vs answerKey.') + + +@register_eval('hellaswag') +def hellaswag(): + return _mcq('hellaswag', 'HellaSwag; letter vs label (acc_norm needs logprobs: model layer).') + + +@register_eval('winogrande') +def winogrande(): + return _mcq('winogrande', 'Winogrande; letter vs answer.') + + +@register_eval('gpqa_diamond') +def gpqa_diamond(): + return _mcq('gpqa_diamond', 'GPQA diamond; letter vs target (choices shuffled at eval time).') + + +@register_eval('longbench_v2') +def longbench_v2(): + return _mcq('longbench_v2', 'LongBench v2; letter vs answer.') diff --git a/evalharness/eval/recipes/qa.py b/evalharness/eval/recipes/qa.py new file mode 100644 index 0000000..7b7dff5 --- /dev/null +++ b/evalharness/eval/recipes/qa.py @@ -0,0 +1,61 @@ +"""QA benchmarks with text-compare scoring: bbh, drop, trivia_qa, openai_mrcr.""" + +import re + +from ..recipe import EvalRecipe, register_eval + + +def _bbh_extract(raw, sample): + """Dispatch by TARGET FORMAT (robust): '(B)' -> MC letter, else free-form phrase. + + The official BBH answers are either '(A)'-style or short free-form text + (True/False, numbers, sorted lists); dispatching on the target's shape + needs no per-subtask table and cannot drift from the data. + """ + from ..extractor import answer_phrase, mcq_letter + + target = str(sample.target or '').strip() + if re.fullmatch(r'\([A-Z]\)', target): + return mcq_letter(raw, sample) + return answer_phrase(raw, sample) + + +@register_eval('bbh') +def bbh(): + return EvalRecipe( + name='bbh', + extract=_bbh_extract, + scorers={'acc': {'name': 'exact', 'mode': 'math'}}, + description='BBH; MC subtasks -> letter, free-form -> answer phrase.', + ) + + +@register_eval('drop') +def drop(): + return EvalRecipe( + name='drop', + extract=['answer_spans', 'first_line'], + scorers={'em': 'em_f1', 'f1': 'em_f1'}, + description='DROP; every Answer: line = one span; official Hungarian-align EM/F1.', + ) + + +@register_eval('trivia_qa') +def trivia_qa(): + return EvalRecipe( + name='trivia_qa', + extract=['first_line', 'answer_phrase'], + scorers={'em': 'alias_match'}, + description='TriviaQA; any alias counts.', + ) + + +@register_eval('openai_mrcr') +def openai_mrcr(): + return EvalRecipe( + name='openai_mrcr', + extract=['quoted_list', 'identity'], + scorers={'mrcr_score': 'exact'}, # placeholder scorer; official prefix grading lands with long-context skill + aggregators={'mrcr_score': 'binned_avg'}, + description='MRCR; quoted-marker extraction, binned by context length.', + ) diff --git a/evalharness/eval/record.py b/evalharness/eval/record.py new file mode 100644 index 0000000..2ea7ed8 --- /dev/null +++ b/evalharness/eval/record.py @@ -0,0 +1,69 @@ +"""Evaluation-layer record schemas. + +The eval layer produces immutable artifacts: for every sample we keep the +RAW prediction (never the extracted string alone) so any recipe change can +re-score without re-running the model. Aggregated reports carry the recipe +fingerprint for reproducibility. + +Data flow: Dataset x predictions -> SampleResult per sample + -> SampleResult list -> EvalReport (aggregates + artifacts) +""" + +import time +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +class SampleResult(BaseModel): + """One evaluated sample. raw_prediction is the source of truth.""" + + sample_id: Optional[int] = None + dataset: str = '' + subset: str = '' + task_type: Optional[str] = None + + raw_prediction: str = '' # exactly what the model produced + extracted_prediction: str = '' # extractor output (derivable) + extraction_ok: bool = True # False => extractor found nothing usable + extraction_note: str = '' # e.g. which cascade stage hit + + scores: Dict[str, float] = Field(default_factory=dict) # {'acc': 1.0, 'f1': 0.6} + score_details: Dict[str, Any] = Field(default_factory=dict) # {'acc': {'judge_raw': 'GRADE: C'}} + + target: Union[str, List[str]] = '' + group_key: str = '' # pass@k task id / binned bucket / category + metadata: Dict[str, Any] = Field(default_factory=dict) + error: str = '' # scorer/extractor exception (never silently dropped) + + +class EvalReport(BaseModel): + """Aggregated artifact: what a viewer/visualizer consumes.""" + + dataset: str + recipe: str = '' # e.g. 'gsm8k' + recipe_version: str = '' + model: str = '' # model name/url tag + created_at: str = Field(default_factory=lambda: time.strftime('%Y-%m-%d %H:%M:%S')) + + num_samples: int = 0 + num_failed_extractions: int = 0 + metrics: Dict[str, float] = Field(default_factory=dict) # {'acc': 0.62} + metric_groups: Dict[str, Dict[str, float]] = Field(default_factory=dict) + # {'by_category': {'algebra': 0.7, ...}, 'pass_at_k': {'pass@1': .., 'pass@8': ..}, + # 'by_length_bin': {'8k': .., '32k': ..}} + + samples: List[SampleResult] = Field(default_factory=list) # full per-sample detail + + def save(self, path) -> None: + import json + + with open(path, 'w', encoding='utf-8') as f: + json.dump(self.model_dump(), f, ensure_ascii=False, indent=2) + + @classmethod + def load(cls, path) -> 'EvalReport': + import json + + with open(path, encoding='utf-8') as f: + return cls.model_validate(json.load(f)) diff --git a/evalharness/eval/registry.py b/evalharness/eval/registry.py new file mode 100644 index 0000000..80df6fd --- /dev/null +++ b/evalharness/eval/registry.py @@ -0,0 +1,34 @@ +"""Shared tiny registry for eval-layer plugins (mirrors data registry semantics).""" + +import difflib +from typing import Callable, Dict, List + + +class EvalRegistry: + """Dict registry: decorator registration, duplicate guard, suggestions.""" + + def __init__(self, kind: str): + self.kind = kind + self._items: Dict[str, Callable] = {} + + def register(self, name: str, fn: Callable) -> Callable: + if name in self._items: + raise ValueError(f'{self.kind} {name!r} is already registered') + self._items[name] = fn + return fn + + def get(self, name: str) -> Callable: + if name not in self._items: + near = difflib.get_close_matches(name, self._items, n=3) + hint = f" Did you mean: {', '.join(near)}?" if near else '' + raise KeyError(f'unknown {self.kind} {name!r}.{hint}') + return self._items[name] + + def names(self) -> List[str]: + return sorted(self._items) + + def __contains__(self, name: str) -> bool: + return name in self._items + + def __len__(self) -> int: + return len(self._items) diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py new file mode 100644 index 0000000..94ee648 --- /dev/null +++ b/evalharness/eval/runner.py @@ -0,0 +1,125 @@ +"""The evaluation runner: Dataset x predictions -> EvalReport. + +Pure orchestration, no I/O hidden inside: predictions arrive as a list +(loaded from a jsonl of model outputs, a Session store, or built inline), +results aggregate into an EvalReport that visualizers consume. + +Judge wiring: pass judge=str> once a ModelAdapter +exists; llm_judge recipes work immediately after that, no recipe change. +""" + +import traceback +from typing import Callable, Dict, Iterable, List, Optional, Sequence, Union + +from ..data.dataset import Dataset +from ..data.sample import Sample +from .aggregator import mean as _mean_agg +from .recipe import EvalRecipe +from .record import EvalReport, SampleResult +from .scorer import ScoreContext + + +def evaluate( + dataset: Union[Dataset, List[Sample]], + predictions: Sequence[Union[str, Dict]], + recipe: Optional[EvalRecipe] = None, + *, + model: str = '', + judge: Optional[Callable] = None, + extra_metadata: Optional[Dict] = None, +) -> EvalReport: + """Score a dataset against raw predictions. + + dataset: a Dataset or a plain list of Samples (views/slices). + predictions: str per sample (raw model output) or dicts with + {'raw': str, 'group_key': ..., 'metadata': {...}} overrides. + """ + samples: List[Sample] = list(dataset) + spec = getattr(dataset, 'spec', None) + ds_name = spec.name if spec is not None else samples[0].metadata.get('dataset', 'adhoc') if samples else 'adhoc' + ds_subset = spec.subset if spec is not None else '' + if len(predictions) != len(samples): + raise ValueError(f'{len(predictions)} predictions for {len(samples)} samples') + + if recipe is None: + from .recipe import get_eval + + recipe = get_eval(ds_name) + extractor = recipe.resolve_extract() + scorers = recipe.resolve_scorers() + aggregators = recipe.resolve_aggregators() + ctx = ScoreContext(judge=judge, params={}) + + results: List[SampleResult] = [] + for sample, pred in zip(samples, predictions): + raw = pred if isinstance(pred, str) else str(pred.get('raw', '')) + override = {} if isinstance(pred, str) else pred + result = SampleResult( + sample_id=sample.id, + dataset=ds_name, + subset=ds_subset, + task_type=sample.task_type, + raw_prediction=raw, + target=sample.target, + group_key=override.get('group_key') + or sample.metadata.get('group_key') + or (sample.metadata.get('task_id') or sample.metadata.get('id') or ''), + metadata={k: v for k, v in (sample.metadata or {}).items() + if k in ('category', 'subject', 'test_category', 'bin', 'difficulty')}, + ) + if isinstance(pred, dict) and pred.get('metadata'): + result.metadata.update(pred['metadata']) + try: + value, ok, note = extractor(raw, sample) + result.extracted_prediction = value + result.extraction_ok = ok + result.extraction_note = note + if not ok: + result.extraction_note = note or 'extractor returned not-ok' + for metric, scorer in scorers.items(): + try: + scores, details = scorer(value if ok else '', sample.target, sample, ctx) + result.scores.update(scores) + result.score_details.update(details) + except Exception as e: # one metric failing must not kill the run + result.scores[metric] = 0.0 + result.score_details[metric] = {'error': f'{type(e).__name__}: {e}'} + except Exception as e: + result.error = f'{type(e).__name__}: {e}\n{traceback.format_exc(limit=2)}' + results.append(result) + + report = EvalReport( + dataset=ds_name, + recipe=recipe.name or dataset.spec.name, + model=model, + num_samples=len(results), + num_failed_extractions=sum(1 for r in results if not r.extraction_ok), + samples=results, + ) + _aggregate_into(report, results, recipe, aggregators) + if extra_metadata: + report.metric_groups['run_info'] = {k: v for k, v in extra_metadata.items() + if isinstance(v, (int, float, str))} + return report + + +def _aggregate_into(report: EvalReport, results, recipe: EvalRecipe, aggregators) -> None: + for metric in recipe.scorers: + agg = aggregators.get(metric) + if agg is None: + agg = _mean_agg + try: + out = agg(results, metric) + except Exception as e: + report.metric_groups[f'agg_error_{metric}'] = {'error': str(e)[:200]} + continue + if isinstance(out, dict): + report.metric_groups[metric] = out + vals = [v for v in out.values() if isinstance(v, (int, float))] + if vals: + report.metrics[metric] = sum(vals) / len(vals) + else: + report.metrics[metric] = float(out) + report.metrics['extraction_failure_rate'] = ( + report.num_failed_extractions / report.num_samples if report.num_samples else 0.0 + ) diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py new file mode 100644 index 0000000..2ae903c --- /dev/null +++ b/evalharness/eval/scorer.py @@ -0,0 +1,323 @@ +"""Scorer primitives: compare an extracted prediction against the target. + +Contract: fn(pred: str, target, sample: Sample, ctx: ScoreContext) -> (scores, details) + scores: {'acc': 1.0} details: {'acc': {...audit info...}} +Register: @register_scorer('exact') +Look up: get_scorer('exact') / make_scorers({'acc': 'exact', ...}) + +Four scoring paradigms (mirrors the benchmark survey): + text compare -- exact / math_equal / em_f1 / alias_match [implemented] + LLM-as-judge -- llm_judge [needs ModelAdapter; wired via ctx.judge] + code execution -- execution [needs Sandbox layer; raises NotReady] + environment reward-- env_reward [needs Agent loop; raises NotReady] +""" + +import re +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from pydantic import BaseModel + +from ..data.sample import Sample +from .registry import EvalRegistry + +ScorerFn = Callable[[str, Any, Sample, 'ScoreContext'], Tuple[Dict[str, float], Dict[str, Any]]] + +SCORER_REGISTRY = EvalRegistry('scorer') + + +def register_scorer(name: str): + def decorator(fn: ScorerFn) -> ScorerFn: + SCORER_REGISTRY.register(name, fn) + return fn + + return decorator + + +def get_scorer(name: str) -> ScorerFn: + return SCORER_REGISTRY.get(name) + + +class ScoreContext(BaseModel): + """Everything a scorer may need beyond (pred, target, sample). + + judge: optional callable(prompt_messages) -> str. Provided by the runner + when a ModelAdapter is configured; llm_judge scorers raise NotReady if + it is None (explicit, never silently wrong). + """ + + class Config: + arbitrary_types_allowed = True + + judge: Optional[Callable] = None + judge_model: str = '' + params: Dict[str, Any] = {} + + +class LayerNotReady(RuntimeError): + """A scoring paradigm needs a layer that is not built yet (sandbox/agent).""" + + +# ------------------------- normalization helpers ------------------------- + + +def _strip_string(s: str) -> str: + """Light math normalization (subset of Hendrycks/Qwen strip_string).""" + s = s.strip() + s = re.sub(r'\\text\{(.+?)\}', r'\1', s) + s = re.sub(r'\\!|\\,|\\;|\\ ', '', s) + s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '') + s = s.replace('^{\\circ}', '').replace('^\\circ', '') + s = re.sub(r'(\d),(\d{3})', r'\1\2', s) + s = re.sub(r'\.0+(?=$|[^0-9])', '', s) + if len(s) > 1 and s[0] == '{' and s[-1] == '}': + s = s[1:-1] + s = s.replace(' ', '').lower() + return s + + +def normalize_text(value: str, mode: str = 'math') -> str: + if mode == 'raw': + return (value or '').strip() + if mode == 'numeric': + v = _strip_string(value) + try: + f = float(v.replace(',', '')) + return str(int(f)) if f == int(f) else str(f) + except ValueError: + return v + return _strip_string(value) + + +def _targets_list(target) -> List[str]: + if target is None: + return [] + if isinstance(target, list): + return [str(t) for t in target] + return [str(target)] + + +# ------------------------- text-compare scorers ------------------------- + + +@register_scorer('exact') +def exact(pred: str, target, sample: Sample, ctx: ScoreContext): + """Exact match after shared normalization. mode: raw|math|numeric.""" + mode = ctx.params.get('mode', 'math') + norm = normalize_text(pred or '', mode) + hit = int(any(norm == normalize_text(t, mode) for t in _targets_list(target))) + return {'acc': float(hit)}, {'acc': {'mode': mode, 'normalized_pred': norm}} + + +@register_scorer('math_equal') +def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext): + """Normalized equality, then optional sympy symbolic equivalence. + + sympy path is tried only when available and only for non-identical + strings; identical normalized strings short-circuit (fast + exact). + """ + targets = _targets_list(target) + norm = _strip_string(pred or '') + if norm and any(norm == _strip_string(t) for t in targets): + return {'acc': 1.0}, {'acc': {'path': 'normalized_string'}} + + details: Dict[str, Any] = {'acc': {'path': 'none', 'normalized_pred': norm}} + if ctx.params.get('sympy', True): + try: + from .math_grader import grade_answer # lazily imported, optional dep + + if any(grade_answer(pred or '', t) for t in targets): + details['acc'] = {'path': 'sympy'} + return {'acc': 1.0}, details + except ImportError: + details['acc']['sympy'] = 'not installed (pip install sympy pylatexenc)' + return {'acc': 0.0}, details + + +def _token_bag(text: str) -> List[str]: + from string import punctuation + + text = (text or '').lower() + text = re.sub(r'\b(a|an|the)\b', ' ', text) + text = re.sub(f'[{re.escape(punctuation)}]', ' ', text) + return [t for t in text.split() if t] + + +def _drop_normalize(text: str) -> List[str]: + """Official DROP normalization: tokenize on space/hyphen, per-token number + normalization (float str), punctuation strip, article strip, lowercase.""" + from string import punctuation + + out = [] + for token in re.split(r'[ |-]', text or ''): + token = token.lower() + if _is_number_official(token): + token = str(float(token)) + else: + token = ''.join(c for c in token if c not in punctuation) + token = re.sub(r'\b(a|an|the)\b', ' ', token) + token = ' '.join(token.split()) + if token: + out.append(token) + return out + + +def _is_number_official(text: str) -> bool: + try: + float(text) + return True + except ValueError: + return False + + +def _drop_f1(pred_set: set, gold_set: set) -> float: + intersection = len(gold_set & pred_set) + precision = intersection / len(pred_set) if pred_set else 1.0 + recall = intersection / len(gold_set) if gold_set else 1.0 + if precision == 0.0 and recall == 0.0: + return 0.0 + return 2 * precision * recall / (precision + recall) + + +def _drop_metrics(predicted: List[str], gold: List[str]): + """Official get_drop_metrics: EM on normalized span sets; F1 via optimal + 1-1 bag alignment (Hungarian), numbers must intersect.""" + pred_norm = [_drop_normalize(p) for p in predicted if (p or '').strip()] + gold_norm = [_drop_normalize(g) for g in gold if (g or '').strip()] + if not gold_norm: + return 0.0, 0.0 + # EM compares the SET of normalized spans (order-insensitive) + em = 1.0 if pred_norm and set(tuple(p) for p in pred_norm) == set(tuple(g) for g in gold_norm) else 0.0 + + pred_bags = [set(' '.join(p).split()) for p in pred_norm] + gold_bags = [set(' '.join(g).split()) for g in gold_norm] + try: + import numpy as np + from scipy.optimize import linear_sum_assignment + except ImportError as e: + raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[exec]'") from e + + n, m = len(gold_bags), len(pred_bags) + if m == 0: + return 0.0, 0.0 + score = np.zeros((n, m)) + for gi, gb in enumerate(gold_bags): + for pi, pb in enumerate(pred_bags): + gold_nums = {w for w in gb if _is_number_official(w)} + pred_nums = {w for w in pb if _is_number_official(w)} + if not gold_nums or (gold_nums & pred_nums): + score[gi, pi] = _drop_f1(pb, gb) + rows, cols = linear_sum_assignment(-score) + per_bag = [0.0] * max(n, m) + for r, c in zip(rows, cols): + per_bag[r] = max(per_bag[r], score[r, c]) + f1 = round(float(np.mean(per_bag)) * 100, 2) + return em, f1 + + +@register_scorer('em_f1') +def em_f1(pred: str, target, sample: Sample, ctx: ScoreContext): + """DROP official EM/F1 over gold spans. + + Target shape: List[str] (multiple spans) or str. The prediction is split + on the official '\\n' separator of multiple predicted spans. + """ + gold: List[str] = [str(t) for t in target] if isinstance(target, list) else [str(target)] + predicted = [p for p in re.split(r'\n', pred or '') if p.strip()] + em, f1 = _drop_metrics(predicted, gold) + return {'em': em / 100.0 if em > 1.0 else em, 'f1': f1 / 100.0}, {'em': {}, 'f1': {'official': True}} + + +@register_scorer('alias_match') +def alias_match(pred: str, target, sample: Sample, ctx: ScoreContext): + """TriviaQA style: normalized pred equals any alias, or alias contained.""" + targets = _targets_list(target) + norm = normalize_text(pred or '', 'math') + hit = 0 + for t in targets: + tn = normalize_text(t, 'math') + if norm == tn or (len(norm) >= 3 and norm in tn): + hit = 1 + break + return {'em': float(hit)}, {'em': {'aliases': len(targets)}} + + +# ------------------------- judge / execution / env (paradigm slots) ------------------------- + + +@register_scorer('llm_judge') +def llm_judge(pred: str, target, sample: Sample, ctx: ScoreContext): + """LLM-as-judge with an explicit label->scores contract. + + params: prompt_template (with {prediction} {target} {question}), + labels: {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, + primary: 'acc' + """ + if ctx.judge is None: + raise LayerNotReady( + "llm_judge needs a judge model; configure a ModelAdapter (runner judge=...) " + 'before running recipes that use it' + ) + template = ctx.params.get('prompt_template', '{prediction}') + prompt = template.format(prediction=pred or '', target=target or '', question=sample.input_text) + raw = ctx.judge([{'role': 'user', 'content': prompt}]) + import json as _json + + try: # judge callable may return a parsed object + raw_text = raw if isinstance(raw, str) else _json.dumps(raw) + except Exception: + raw_text = str(raw) + labels: Dict[str, Dict[str, float]] = ctx.params.get('labels') or {} + upper = (raw_text or '').upper() + chosen = None + for label in labels: + if label.upper() and label.upper() in upper: + chosen = label + break + primary = ctx.params.get('primary', 'acc') + default_label = ctx.params.get('default_label') # e.g. SimpleQA's C on parse failure + if chosen is None and default_label and default_label in labels: + chosen = default_label + if chosen is None: + scores = {k: 0.0 for k in (labels.get(next(iter(labels))) or {})} + return scores, {primary: {'judge_raw': raw_text[:500], 'parse': 'failed'}} + return dict(labels[chosen]), {primary: {'judge_label': chosen, 'judge_raw': raw_text[:500]}} + + +@register_scorer('execution') +def execution(pred: str, target, sample: Sample, ctx: ScoreContext): + """Run code against tests in a sandbox (humaneval/BCB/LCB style).""" + raise LayerNotReady('execution scorer needs the sandbox layer (roadmap: after model layer)') + + +@register_scorer('env_reward') +def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext): + """Score an agent trajectory by environment reward (tau2/swe style).""" + raise LayerNotReady('env_reward scorer needs the agent loop layer') + + +# ------------------------- resolution ------------------------- + +ScorerSpec = Union[str, ScorerFn, Dict[str, Any]] + + +def make_scorer(metric: str, spec: ScorerSpec) -> ScorerFn: + """Resolve one metric's scorer spec (name / fn / {'name', **params}).""" + params: Dict[str, Any] = {} + if isinstance(spec, dict): + spec = dict(spec) + params = {k: v for k, v in spec.items() if k != 'name'} + spec = spec.get('name') + if callable(spec): + return spec + if isinstance(spec, str): + base = get_scorer(spec) + if not params: + return base + + def with_params(pred, target, sample, ctx): + merged = ScoreContext(judge=ctx.judge, judge_model=ctx.judge_model, + params={**ctx.params, **params}) + return base(pred, target, sample, merged) + + return with_params + raise TypeError(f'bad scorer spec for {metric!r}: {spec!r}') diff --git a/evalharness/viz/__init__.py b/evalharness/viz/__init__.py new file mode 100644 index 0000000..b0d84e3 --- /dev/null +++ b/evalharness/viz/__init__.py @@ -0,0 +1,63 @@ +"""Visualization layer: consume EvalReport artifacts, render views. + +Strictly a CONSUMER of the eval layer (reads saved report json, never scores). +Renderers are registered plugins: text table, markdown, and ascii bar/radar +charts today; a web renderer can register the same way later. + + from evalharness.viz import render + render('report.json', style='text') # console table + render([r1, r2], style='md_compare') # benchmark comparison table +""" + +import importlib +import pkgutil +from pathlib import Path +from typing import Callable, Dict, List, Union + +from ..eval.record import EvalReport + +RendererFn = Callable[[Union['EvalReport', List['EvalReport'], str, Path], Dict], str] + +RENDERERS: Dict[str, RendererFn] = {} + + +def register_renderer(name: str): + def decorator(fn: RendererFn) -> RendererFn: + if name in RENDERERS: + raise ValueError(f'renderer {name!r} already registered') + RENDERERS[name] = fn + return fn + + return decorator + + +def get_renderer(name: str) -> RendererFn: + if name not in RENDERERS: + raise KeyError(f"unknown renderer {name!r}. Available: {', '.join(sorted(RENDERERS))}") + return RENDERERS[name] + + +def render(target, style: str = 'text', **opts) -> str: + """Render one report path/object or a list of them (comparison styles).""" + return get_renderer(style)(_load(target), opts) + + +def _load(target) -> Union[EvalReport, List[EvalReport]]: + if isinstance(target, (str, Path)): + return EvalReport.load(target) + if isinstance(target, EvalReport): + return target + if isinstance(target, (list, tuple)): + return [_load(t) for t in target] + raise TypeError(f'cannot load report from {type(target)}') + + +def _discover_builtin_renderers() -> None: + pkg_dir = Path(__file__).parent / 'renderers' + if not pkg_dir.exists(): + return + for info in pkgutil.iter_modules([str(pkg_dir)]): + importlib.import_module(f'{__name__}.renderers.{info.name}') + + +_discover_builtin_renderers() diff --git a/evalharness/viz/renderers/__init__.py b/evalharness/viz/renderers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evalharness/viz/renderers/text.py b/evalharness/viz/renderers/text.py new file mode 100644 index 0000000..9bd420e --- /dev/null +++ b/evalharness/viz/renderers/text.py @@ -0,0 +1,145 @@ +"""Text/markdown renderers + unicode bar & radar charts (zero dependencies).""" + +import math +from typing import Dict, List, Union + +from ...eval.record import EvalReport +from .. import register_renderer + + +def _bars(value: float, width: int = 30, char: str = '█') -> str: + filled = int(round(max(0.0, min(1.0, value)) * width)) + return char * filled + '·' * (width - filled) + + +def _pct(value: float) -> str: + return f'{value * 100:.1f}%' + + +@register_renderer('text') +def text_table(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: + reports = target if isinstance(target, list) else [target] + out: List[str] = [] + for rep in reports: + head = f'{rep.dataset} [{rep.recipe}] model={rep.model or "?"} n={rep.num_samples}' + out.append('=' * max(len(head), 40)) + out.append(head) + out.append('=' * max(len(head), 40)) + if rep.num_failed_extractions: + warn = (f'!! {rep.num_failed_extractions}/{rep.num_samples} extractions failed ' + f'({_pct(rep.metrics.get("extraction_failure_rate", 0))}) -- check recipe/model fit') + out.append(warn) + for metric, value in rep.metrics.items(): + if metric == 'extraction_failure_rate': + continue + out.append(f'{metric:<16} {_pct(value):>7} {_bars(value)}') + for group_name, groups in rep.metric_groups.items(): + if group_name == 'run_info' or group_name.startswith('agg_error'): + continue + out.append(f'-- {group_name} ' + '-' * max(0, 30 - len(group_name))) + for g, v in groups.items(): + if isinstance(v, (int, float)): + out.append(f' {g:<28} {_pct(v):>7} {_bars(v, 20)}') + out.append('') + return '\n'.join(out) + + +@register_renderer('md') +def markdown(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: + reports = target if isinstance(target, list) else [target] + out = ['# Eval Report', ''] + for rep in reports: + out += [f'## {rep.dataset} (`{rep.recipe}`)', '', + f'- model: `{rep.model or "?"}` samples: {rep.num_samples} ' + f'created: {rep.created_at}', ''] + rows = ['| metric | value |', '|---|---|'] + for metric, value in rep.metrics.items(): + rows.append(f'| {metric} | {_pct(value) if metric != "extraction_failure_rate" else _pct(value)} |') + out += rows + [''] + for gname, groups in rep.metric_groups.items(): + if gname in ('run_info',) or gname.startswith('agg_error') or not isinstance(groups, dict): + continue + out += [f'### {gname}', '', '| group | value |', '|---|---|'] + out += [f'| {g} | {_pct(v) if isinstance(v, (int, float)) else v} |' for g, v in groups.items()] + out.append('') + return '\n'.join(out) + + +@register_renderer('md_compare') +def md_compare(target: List[EvalReport], opts: Dict) -> str: + """Side-by-side metric table for N reports (e.g. two models on one bench).""" + if not isinstance(target, list) or len(target) < 1: + raise ValueError('md_compare needs a list of reports') + metrics: List[str] = [] + for rep in target: + for m in rep.metrics: + if m not in metrics and m != 'extraction_failure_rate': + metrics.append(m) + cols = [f'{rep.dataset}/{rep.recipe}[{rep.model or "?"}]' for rep in target] + out = ['# Comparison', '', '| metric | ' + ' | '.join(cols) + ' |', + '|---' * (len(cols) + 1) + '|'] + for m in metrics: + cells = [_pct(rep.metrics.get(m, 0.0)) for rep in target] + best = max(rep.metrics.get(m, 0.0) for rep in target) + cells = [f'**{c}**' if rep.metrics.get(m, 0.0) == best and len(target) > 1 else c + for c, rep in zip(cells, target)] + out.append(f'| {m} | ' + ' | '.join(cells) + ' |') + out.append('') + return '\n'.join(out) + + +@register_renderer('radar') +def radar(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: + """Unicode radar chart over each report's metric_groups entries. + + opts: group (default: first non-run_info group), top (default 10 axes). + """ + reports = target if isinstance(target, list) else [target] + group = opts.get('group') + axes: List[str] = [] + series: List[Dict[str, float]] = [] + for rep in reports: + gname = group or next((k for k in rep.metric_groups + if k != 'run_info' and not k.startswith('agg_error') + and isinstance(rep.metric_groups[k], dict)), None) + data = rep.metric_groups.get(gname, {}) if gname else {} + data = {k: v for k, v in data.items() if isinstance(v, (int, float))} + if not data: + return f'(no grouped metrics to chart for {rep.dataset})' + top = opts.get('top', 10) + picked = sorted(data.items(), key=lambda kv: -kv[1])[:top] + axes = [k for k, _ in picked] + series.append(dict(picked)) + + # simple ascii radar: axis list + per-report bars side by side + W = 24 + header = 'axis'.ljust(26) + ''.join((rep.dataset[:12] or '?').rjust(14) for rep in reports) + lines = [header, '-' * len(header)] + for ax in axes: + row = ax[:24].ljust(26) + for rep, s in zip(reports, series): + v = s.get(ax, 0.0) + row += (_bars(v, W // 2)[:W // 2] + f'{v * 100:5.1f}%').rjust(14) + lines.append(row) + return '\n'.join(lines) + + +@register_renderer('errors') +def errors(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: + """Failed samples browser: worst-N samples with extraction + target.""" + rep = target[0] if isinstance(target, list) else target + n = opts.get('n', 10) + only_failed = opts.get('only_failed', True) + rows = [r for r in rep.samples if r.error or (not r.extraction_ok if only_failed else False)] + rows = sorted(rows, key=lambda r: sum(r.scores.values()))[:n] + out = [f'# Errors / failed extractions: {rep.dataset} ({len(rows)} shown)', ''] + for r in rows: + out.append(f'## sample {r.sample_id} scores={r.scores}') + out.append(f'- extraction_ok={r.extraction_ok} note={r.extraction_note!r}') + out.append(f'- target: {str(r.target)[:120]!r}') + out.append(f'- extracted: {r.extracted_prediction[:120]!r}') + if r.error: + out.append(f'- error: {r.error[:300]}') + out.append(f'- raw: {r.raw_prediction[:200]!r}') + out.append('') + return '\n'.join(out) diff --git a/pyproject.toml b/pyproject.toml index 8a995fe..31a5d2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,10 @@ requires-python = ">=3.10" dependencies = ["pydantic>=2"] [project.optional-dependencies] -hub = ["datasets"] # needed only for HuggingFace-hosted datasets -parquet = ["pyarrow"] # needed only for parquet sources (ModelScope mirrors) +hub = ["datasets"] # HuggingFace-hosted datasets +parquet = ["pyarrow"] # parquet sources (ModelScope/HF raw mirrors) +math = ["sympy", "pylatexenc"] # symbolic math grading (aime/math family), official PRM800K logic +exec = ["numpy", "scipy"] # DROP official aligner (linear_sum_assignment) & code benches [project.scripts] evalharness = "evalharness.cli:main" diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 0000000..7b90c51 --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,154 @@ +"""Regression tests for the eval layer. + +Anchored to OFFICIAL benchmark grading behavior: +- DROP: allennlp/simple-evals get_drop_metrics test cases +- math: PRM800K grader semantics (sympy equivalence) +- MCQ/gsm8k/BBH: extraction dispatch conventions + +Run: .venv/bin/python -m pytest tests/ -q (or python tests/test_eval.py) +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from evalharness.data.sample import Sample # noqa: E402 +from evalharness.eval import evaluate, get_eval, list_evals # noqa: E402 +from evalharness.eval.extractor import make_extractor # noqa: E402 +from evalharness.eval.scorer import _drop_metrics # noqa: E402 + + +def _mk(input_text='q', target='', **kw): + return Sample(input=input_text, target=target, **kw) + + +def test_drop_official_cases(): + cases = [ + (['test'], ['test'], (1.0, 100.0)), + (['test'], ['testing'], (0.0, 0.0)), # token bags disjoint -> f1 0 (official) + (['test test'], ['test'], (0.0, 100.0)), + (['a'], ['b'], (0.0, 0.0)), + (['1'], ['1.0'], (1.0, 100.0)), + (['1'], ['2'], (0.0, 0.0)), + (['ted', 'dan'], ['dan', 'ted'], (1.0, 100.0)), + (['x'], ['y', 'z'], (0.0, 0.0)), + ] + for pred, gold, want in cases: + got = _drop_metrics(pred, gold) + assert got == want, f'{pred} vs {gold}: {got} != official {want}' + + +def test_math_grader(): + from evalharness.eval.math_grader import grade_answer + + assert grade_answer('0.5', '\\dfrac{1}{2}') + assert not grade_answer('0.6', '\\dfrac{1}{2}') + assert grade_answer('70000', '70,\\!000') + assert not grade_answer('x=2', '2') + assert grade_answer('1/2', '0.5') is True or grade_answer('1/2', '0.5') is False # deterministic + + +def test_extractors(): + box = make_extractor('math_boxed') + assert box('\\boxed{42} done', _mk())[0] == '42' + assert box('no box here', _mk())[1] is False + cascade = make_extractor(['math_boxed', 'answer_phrase', 'last_number']) + assert cascade('The answer is 7.', _mk())[0] == '7' + assert cascade('total 3 apples and 5 pears', _mk())[0] == '5' + letter = make_extractor('mcq_letter') + assert letter('So the answer is (B).', _mk())[0] == 'B' + assert letter('答案是C', _mk())[0] == 'C' + spans = make_extractor('answer_spans') + got, ok, _ = spans('Answer: Chaz Schilens\nAnswer: JaMarcus Russell', _mk()) + assert ok and got == 'Chaz Schilens\nJaMarcus Russell' + + +def test_gsm8k_end_to_end(): + samples = [_mk(str(i), str(10 + i)) for i in range(4)] + preds = ['\\boxed{10}', '#### 11', 'The answer is 12.', 'no idea'] + rep = evaluate(samples, preds, get_eval('gsm8k')) + assert rep.metrics['acc'] == 0.75 + assert rep.num_failed_extractions == 1 + + +def test_bbh_dispatch_by_target_format(): + mc = _mk('q?', '(B)', metadata={'subset': 'date_understanding'}) + ff = _mk('q?', 'True') + ex = get_eval('bbh').resolve_extract() + assert ex('So the answer is (B).', mc)[0] == 'B' + assert ex('the answer is True', ff)[0] == 'True' + + +def test_mcq_end_to_end(): + samples = [_mk('q', 'B', choices=['a', 'b'])] * 2 + rep = evaluate(samples, ['(B)', '答案是B'], get_eval('mmlu')) + assert rep.metrics['acc'] == 1.0 + + +def test_all_recipes_resolve(): + from evalharness.eval.aggregator import get_aggregator + from evalharness.eval.scorer import get_scorer + + for name in list_evals(): + recipe = get_eval(name) + recipe.resolve_extract() + scorers = recipe.resolve_scorers() + assert scorers + recipe.resolve_aggregators() + for spec in recipe.aggregators.values(): + get_aggregator(spec[0] if isinstance(spec, tuple) else (spec or 'mean')) + + +def test_judge_default_label(): + from evalharness.eval.scorer import ScoreContext, get_scorer + + scorer = get_scorer('llm_judge') + ctx = ScoreContext(judge=lambda msgs: 'zzz', params={ # no A/B/C anywhere + 'prompt_template': '{prediction}', + 'labels': {'A': {'is_correct': 1.0}, 'B': {'is_correct': 0.0}, 'C': {'is_correct': 0.0}}, + 'default_label': 'C', 'primary': 'is_correct'}) + scores, details = scorer('x', 'y', _mk('q'), ctx) + assert scores['is_correct'] == 0.0 and details['is_correct']['judge_label'] == 'C' + + +def test_aggregators(): + from evalharness.eval.aggregator import grouped_avg, mean, unbiased_pass_at_k + from evalharness.eval.record import SampleResult + + rs = [SampleResult(raw_prediction='', scores={'acc': 1.0}, group_key='x'), + SampleResult(raw_prediction='', scores={'acc': 0.0}, group_key='x'), + SampleResult(raw_prediction='', scores={'acc': 1.0}, group_key='y')] + assert mean(rs, 'acc') == 2 / 3 + assert grouped_avg(rs, 'acc') == {'x': 0.5, 'y': 1.0} + assert unbiased_pass_at_k(10, 10, 5) == 1.0 + assert abs(unbiased_pass_at_k(10, 5, 5) - (1 - 1 / 252)) < 1e-9 # C(5,5)/C(10,5)=1/252 + + +def test_viz_renders(tmp_path=None): + from evalharness.viz import render + + samples = [_mk('q', '1')] * 3 + rep = evaluate(samples, ['\\boxed{1}'] * 3, get_eval('gsm8k'), model='m1') + text = render(rep, style='text') + assert 'acc' in text and '100.0%' in text + md = render(rep, style='md') + assert '| metric |' in md + errs = render(rep, style='errors') + assert 'Errors' in errs or errs == '' # no errors -> empty-ish + + +if __name__ == '__main__': + fails = 0 + for name, fn in sorted({k: v for k, v in globals().items() + if k.startswith('test_') and callable(v)}.items()): + try: + fn() + print(f'PASS {name}') + except AssertionError as e: + fails += 1 + print(f'FAIL {name}: {e}') + except Exception as e: + fails += 1 + print(f'ERROR {name}: {type(e).__name__}: {e}') + sys.exit(1 if fails else 0)