From f8cd15fea18a7652cb7bf01d47ad7a46568504cf Mon Sep 17 00:00:00 2001 From: sora Date: Mon, 24 Aug 2026 03:35:11 +0000 Subject: [PATCH] EvalHarness data layer: 28 dataset plugins, lazy materialize cache (raw/ + samples.jsonl + meta.json), ModelScope native loader, CLI list/fetch/unload/stats/show --- .gitignore | 6 + README.md | 220 +++++++++++++++ evalharness/__init__.py | 21 ++ evalharness/__main__.py | 5 + evalharness/cli.py | 143 ++++++++++ evalharness/data/__init__.py | 51 ++++ evalharness/data/dataset.py | 239 ++++++++++++++++ evalharness/data/datasets/__init__.py | 6 + evalharness/data/datasets/aime24.py | 29 ++ evalharness/data/datasets/aime25.py | 26 ++ evalharness/data/datasets/aime26.py | 27 ++ evalharness/data/datasets/arc.py | 29 ++ evalharness/data/datasets/bbh.py | 23 ++ evalharness/data/datasets/bfcl_v3.py | 63 +++++ evalharness/data/datasets/bigcodebench.py | 32 +++ evalharness/data/datasets/cmmlu.py | 50 ++++ evalharness/data/datasets/competition_math.py | 44 +++ evalharness/data/datasets/drop.py | 31 ++ evalharness/data/datasets/general_fc.py | 34 +++ evalharness/data/datasets/gpqa_diamond.py | 43 +++ evalharness/data/datasets/gsm8k.py | 33 +++ evalharness/data/datasets/hellaswag.py | 29 ++ evalharness/data/datasets/hle.py | 37 +++ evalharness/data/datasets/hmmt26.py | 27 ++ evalharness/data/datasets/humaneval.py | 30 ++ evalharness/data/datasets/imo_answerbench.py | 32 +++ evalharness/data/datasets/live_code_bench.py | 38 +++ evalharness/data/datasets/longbench_v2.py | 33 +++ evalharness/data/datasets/mmlu.py | 30 ++ evalharness/data/datasets/mmlu_pro.py | 27 ++ evalharness/data/datasets/openai_mrcr.py | 32 +++ evalharness/data/datasets/simple_qa.py | 29 ++ .../data/datasets/swe_bench_verified.py | 38 +++ evalharness/data/datasets/tau2_bench.py | 46 +++ evalharness/data/datasets/trivia_qa.py | 29 ++ evalharness/data/datasets/winogrande.py | 27 ++ evalharness/data/loader.py | 265 ++++++++++++++++++ evalharness/data/registry.py | 93 ++++++ evalharness/data/sample.py | 62 ++++ evalharness/data/spec.py | 38 +++ pyproject.toml | 23 ++ 41 files changed, 2120 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 evalharness/__init__.py create mode 100644 evalharness/__main__.py create mode 100644 evalharness/cli.py create mode 100644 evalharness/data/__init__.py create mode 100644 evalharness/data/dataset.py create mode 100644 evalharness/data/datasets/__init__.py create mode 100644 evalharness/data/datasets/aime24.py create mode 100644 evalharness/data/datasets/aime25.py create mode 100644 evalharness/data/datasets/aime26.py create mode 100644 evalharness/data/datasets/arc.py create mode 100644 evalharness/data/datasets/bbh.py create mode 100644 evalharness/data/datasets/bfcl_v3.py create mode 100644 evalharness/data/datasets/bigcodebench.py create mode 100644 evalharness/data/datasets/cmmlu.py create mode 100644 evalharness/data/datasets/competition_math.py create mode 100644 evalharness/data/datasets/drop.py create mode 100644 evalharness/data/datasets/general_fc.py create mode 100644 evalharness/data/datasets/gpqa_diamond.py create mode 100644 evalharness/data/datasets/gsm8k.py create mode 100644 evalharness/data/datasets/hellaswag.py create mode 100644 evalharness/data/datasets/hle.py create mode 100644 evalharness/data/datasets/hmmt26.py create mode 100644 evalharness/data/datasets/humaneval.py create mode 100644 evalharness/data/datasets/imo_answerbench.py create mode 100644 evalharness/data/datasets/live_code_bench.py create mode 100644 evalharness/data/datasets/longbench_v2.py create mode 100644 evalharness/data/datasets/mmlu.py create mode 100644 evalharness/data/datasets/mmlu_pro.py create mode 100644 evalharness/data/datasets/openai_mrcr.py create mode 100644 evalharness/data/datasets/simple_qa.py create mode 100644 evalharness/data/datasets/swe_bench_verified.py create mode 100644 evalharness/data/datasets/tau2_bench.py create mode 100644 evalharness/data/datasets/trivia_qa.py create mode 100644 evalharness/data/datasets/winogrande.py create mode 100644 evalharness/data/loader.py create mode 100644 evalharness/data/registry.py create mode 100644 evalharness/data/sample.py create mode 100644 evalharness/data/spec.py create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0dc5bbd --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.venv/ +dist/ +*.egg-info/ +__pycache__/ +*.pyc +.DS_Store diff --git a/README.md b/README.md new file mode 100644 index 0000000..5a5bbd4 --- /dev/null +++ b/README.md @@ -0,0 +1,220 @@ +# EvalHarness + +A plugin-based LLM/agent evaluation harness. **Currently: the data layer only** +(dataset registration + lazy materialization cache + CLI). Model/eval/sandbox/ +tool/skill layers are intentionally not built yet — they land one at a time. + +## Features + +- **Unified `Sample` schema** (pydantic): `input / choices / target / task_type / + tools / sandbox / files / setup / metadata`. Raw dataset formats are + unconstrained; each dataset plugin converts its records into `Sample`. +- **Dataset registration**: `@register_dataset(DatasetSpec(...))` decorator, + import-time registration; `get_dataset(name)` returns a **lazy handle** — + listing and registration never download anything. +- **Lazy materialization + content-addressed cache**: the first real use + (iteration / `len` / indexing) triggers `download -> convert -> cache write`. + Cache dir layout: `datasets//_[-]-/` + — readable benchmark folders, one readable subdir per subset/split/version. + A file lock prevents duplicate concurrent downloads; `tmp+rename` atomic + writes prevent torn caches. +- **Two plugin styles**: pure `FieldSpec` declarative mapping when records are + well-shaped (zero conversion code), or a custom `record_to_sample` function. +- **CLI**: `list / fetch (concurrent) / unload / stats / show`. +- **28 built-in datasets** registered against their official sources + (see table below). + +## Install + +```bash +pip install . # from this repo; core only (pydantic) +pip install '.[hub]' # + HuggingFace `datasets`, needed for hub sources +``` + +For development: `pip install -e '.[hub]'`. + +To build and install the wheel yourself: + +```bash +pip install build +python -m build # produces dist/evalharness-0.1.0-py3-none-any.whl + .tar.gz +pip install dist/evalharness-0.1.0-py3-none-any.whl +``` + +> Publishing to PyPI (`twine upload dist/*`) makes `pip install evalharness` +> work for everyone — note the name may be taken, so pick a unique +> distribution name (e.g. `evalharness-x`) in `pyproject.toml` before upload. + +## Quick start + +```bash +evalharness data list # list registered datasets (no network, no download) +evalharness data fetch gsm8k # materialize: first run from source, then cache +evalharness data fetch gsm8k mmlu arc --workers 8 # concurrent prefetch +evalharness data unload gsm8k # drop the cache entry (raw/ + samples + meta) +evalharness data stats cmmlu # materialize + stats (count/lengths/answers/cache path) +evalharness data show gsm8k -n 2 # print the first N samples + +# spec overrides: offline demo / local data / picking a subset +evalharness data fetch gsm8k --source examples/data/gsm8k_main_test.jsonl # bundled tiny set +evalharness data fetch mmlu --subset anatomy # one of MMLU's 57 subjects +evalharness data fetch bbh --subset word_sorting # one of BBH's 27 subtasks +# note: with multiple names, --source/--split/--subset apply to ALL of them; +# run separately when you need per-benchmark overrides +``` + +Python API: + +```python +from evalharness import get_dataset, list_datasets + +ds = get_dataset('gsm8k') # lazy handle: zero network/disk cost here +len(ds) # first use -> materialize (download->convert->cache) +for s in ds: + print(s.input, s.target) + +ds_sub = get_dataset('mmlu', subset='anatomy') # spec override, separate cache +hard = ds.view([s for s in ds if len(s.input_text) > 100], + lineage={'tool': 'length_filter'}) # derived data: same class, with lineage +``` + +Cache root: `~/.cache/evalharness/` (override with the `EVALHARNESS_CACHE` +environment variable). Layout example: + +``` +datasets/ +├── gsm8k/ +│ ├── main_test-e06f82/ <- official openai/gsm8k +│ │ ├── raw/ # NATIVE source data, byte-exact as downloaded +│ │ ├── samples.jsonl # converted unified Sample view +│ │ └── meta.json # spec + provenance + raw file list +│ └── main_test-daafcc/ <- local demo via --source +├── mmlu/ +│ ├── all_test-1a3b4c/ +│ └── anatomy_test-9e666f/ +└── bbh/ + └── boolean_expressions_test-… (one dir per subtask) + +.raw// shared download blobs (ModelScope sources); + cache entries hardlink from here, so multi- + subset mirrors download only once +``` + +> Every cache entry is **self-contained and preserves native data**: `raw/` +> holds the original file(s) exactly as downloaded (never converted); +> `samples.jsonl` is the derived unified view. HF-hub sources keep an exact +> pre-conversion record dump in `raw/records.jsonl`. Rebuild any entry with +> `evalharness data fetch --force`. + +> Why the 6-char hash suffix: two variants with the same subset/split but +> different sources (`--source`) or params would otherwise collide and serve +> stale data. The short hash keeps them apart while staying readable. + +## Built-in datasets (28, official sources) + +| Family | Datasets (source) | +|---|---| +| Math | gsm8k (openai/gsm8k), competition_math (EleutherAI/hendrycks_math), aime24 (HuggingFaceH4), aime25 (yentinglin), aime26*, hmmt26*, imo_answerbench* | +| Knowledge / MCQ | mmlu (cais/mmlu), mmlu_pro (TIGER-Lab), cmmlu (haonan-li), gpqa_diamond (Idavidrein/gpqa), arc (allenai/ai2_arc), hellaswag, winogrande | +| QA | trivia_qa (mandarjoshi), drop (ucinlp), simple_qa (mirror of OpenAI's CSV), hle (cais/hle), bbh (lukaemon/bbh) | +| Long context | longbench_v2 (THUDM), openai_mrcr (openai-mirror) | +| Coding | humaneval (openai), bigcodebench (bigcode), live_code_bench (livecodebench) | +| Agent / tools | swe_bench_verified (princeton-nlp), tau2_bench (official GitHub), bfcl_v3 (official GitHub), general_fc (evalscope-native) | + +\* aime26 / hmmt26 / imo_answerbench have no official standalone release and +use community curations (evalscope); simple_qa's official artifact is the CSV +in `openai/simple-evals` (HF is a mirror); tau2_bench / bfcl_v3 are officially +released on GitHub — clone and point `--source` at the local files. + +## Adding a dataset + +Drop a single-file plugin into `evalharness/data/datasets/` — auto-discovered, +no central file to edit. + +Well-shaped records (column names map directly) — pure declaration: + +```python +# evalharness/data/datasets/cmmlu.py +@register_dataset(DatasetSpec(name='cmmlu', source='haonan-li/cmmlu', split='test', task_type='mcq')) +def cmmlu(): + return FieldSpec(input='question', choices='choices', target='answer', metadata=['category']) +``` + +Custom conversion — return a `record -> Sample` function: + +```python +@register_dataset(DatasetSpec(name='gsm8k', source='openai/gsm8k', subset='main', + split='test', task_type='math')) +def gsm8k(): + def to_sample(record): + parts = record['answer'].split('####') + return Sample(input=record['question'], target=parts.pop().strip()) + return to_sample +``` + +Sources supported: local `.jsonl/.json/.csv/.tsv` files; local directories +(probed as `{subset}_{split}.jsonl` etc.); HF hub dataset ids (requires the +`hub` extra, imported lazily); official GitHub releases (clone, then +`--source` the local path). + +Conventions: `choices` holds option *contents*; `target` is a **letter** for +MCQ (e.g. `'B'`) and text otherwise (use `List[str]` for multiple gold +answers); long contexts/passages go to `metadata`, not `input`; `id` is +assigned sequentially at materialize time when absent. + +## Design decisions + +The data layer mirrors conclusions from a close reading of seven evaluation +frameworks (evalscope, lm-evaluation-harness, inspect_ai, deepeval, VLMEvalKit, +harbor, deepseek-harness); see `../README.md` for the full analysis: + +- **`Sample` base + one `record_to_sample` per plugin** (evalscope/inspect_ai): + raw formats vary wildly; the unified format only exists after conversion, + and everything downstream sees only `Sample`. +- **Datasets as first-class citizens** (inspect_ai): a `Dataset` from + `get_dataset()` can be filtered/synthesized/exported freely — it is not + welded into an evaluation recipe. +- **Registration is cheap, materialization pays**: the registry holds only + metadata + conversion recipes; `list` never touches the network. +- **Config-sensitive cache keys** (evalscope): a cache hit is always correct; + no invalidation logic exists. +- **Sandbox/tool fields on `Sample` are declarations only** + (`sandbox/files/setup/tools`): the data layer never executes; a future + sandbox layer will materialize them. +- **Data load/unload vs environment load/unload are different layers**: + `fetch`/`unload` move bytes only (raw files + converted samples). Heavy + execution environments (e.g. the ~1GB-per-instance `sweb.eval.*` images + declared by swe_bench_verified) are pulled lazily *at eval time* by the + sandbox layer — never at data-fetch time — and their removal is + refcounted there because docker layers are shared across instances and + benchmarks. `DatasetSpec.requires` (e.g. `['docker']`) is the declaration + the sandbox/deploy layer reads. + +## Layout + +``` +EvalHarness/ +├── pyproject.toml +├── evalharness/ +│ ├── cli.py # argparse CLI: data list/fetch(concurrent)/stats/show +│ └── data/ +│ ├── sample.py # Sample / ChatMessage / SandboxSpec / ToolInfo +│ ├── spec.py # DatasetSpec (metadata) / FieldSpec (field mapping) +│ ├── registry.py # Registry + @register_dataset + get_dataset +│ ├── loader.py # raw record loading (local file/dir/hub) +│ ├── dataset.py # Dataset: lazy materialize + cache + derived views +│ └── datasets/ # 28 built-in single-file dataset plugins +├── examples/ +│ └── data/ # offline demo subsets (gsm8k/cmmlu, 5 rows each) +``` + +## Roadmap (not built yet, one layer at a time) + +- [ ] Model layer (ModelAdapter: unified URL-based invocation) +- [ ] Evaluation engine (evaluator + scorer/metric) +- [ ] Sandbox layer (materialize `Sample.sandbox`: lazy per-instance image + pull, refcounted image unload, container lifecycle; `requires` gating) +- [ ] Tool layer (data filter/synthesis/dedup/export; Dataset in, Dataset out) +- [ ] Skill layer (full evaluation pipelines as composable skills) +- [ ] Plugin runtime upgrade (apply/ctx/disposer/inject; today: simple registry) +- [ ] Web/API interface diff --git a/evalharness/__init__.py b/evalharness/__init__.py new file mode 100644 index 0000000..165a512 --- /dev/null +++ b/evalharness/__init__.py @@ -0,0 +1,21 @@ +"""EvalHarness: a plugin-based LLM/agent evaluation harness (data layer first).""" + +from .data import ( + ChatMessage, + Dataset, + DatasetSpec, + FieldSpec, + Sample, + SandboxSpec, + ToolInfo, + get_dataset, + list_datasets, + register_dataset, +) + +__version__ = '0.1.0' + +__all__ = [ + 'Dataset', 'DatasetSpec', 'FieldSpec', 'Sample', 'ChatMessage', 'SandboxSpec', 'ToolInfo', + 'get_dataset', 'list_datasets', 'register_dataset', +] diff --git a/evalharness/__main__.py b/evalharness/__main__.py new file mode 100644 index 0000000..dd8a8c9 --- /dev/null +++ b/evalharness/__main__.py @@ -0,0 +1,5 @@ +import sys + +from .cli import main + +sys.exit(main()) diff --git a/evalharness/cli.py b/evalharness/cli.py new file mode 100644 index 0000000..300f9a2 --- /dev/null +++ b/evalharness/cli.py @@ -0,0 +1,143 @@ +"""EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic).""" + +import argparse +import json +import sys +from concurrent.futures import ThreadPoolExecutor, as_completed + + +def _overrides(args): + """Optional DatasetSpec field overrides shared by fetch/stats/show.""" + if getattr(args, 'cache_dir', None): + from evalharness.data.dataset import set_cache_root + + set_cache_root(args.cache_dir) + ov = {} + for k in ('source', 'split', 'subset'): + v = getattr(args, k, None) + if v is not None: + ov[k] = v + return ov + + +def _cmd_data_list(_args) -> int: + from evalharness.data import list_datasets + + specs = list_datasets() + if not specs: + print('no datasets registered') + return 0 + name_w = max(len(s.name) for s in specs) + type_w = max(len(s.task_type) for s in specs) + for s in specs: + print(f'{s.name:<{name_w}} {s.task_type:<{type_w}} {s.source} {s.description}') + print(f'\n{len(specs)} dataset(s) registered') + return 0 + + +def _fetch_one(name: str, force: bool, overrides) -> str: + from evalharness.data import get_dataset + + ds = get_dataset(name, **overrides) + ds.materialize(force=force) + origin = 'cache' if ds.lineage.get('from') == 'cache' else 'source' + return f'{name}: {len(ds)} sample(s) [{origin}] -> {ds.cache_dir}' + + +def _cmd_data_fetch(args) -> int: + names = args.names + if len(names) == 1: + print(_fetch_one(names[0], args.force, _overrides(args))) + return 0 + # Concurrent prefetch: downloads are I/O-bound, threads suffice. + # Per-dataset file locks inside materialize() guard shared cache entries. + ok = True + with ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = {pool.submit(_fetch_one, n, args.force, _overrides(args)): n for n in names} + for fut in as_completed(futures): + try: + print(fut.result()) + except Exception as e: # one failure must not block the rest + ok = False + print(f'{futures[fut]}: FAILED ({e})', file=sys.stderr) + return 0 if ok else 1 + + +def _cmd_data_stats(args) -> int: + from evalharness.data import get_dataset + + stats = get_dataset(args.name, **_overrides(args)).stats() + print(json.dumps(stats, ensure_ascii=False, indent=2)) + return 0 + + +def _cmd_data_show(args) -> int: + from evalharness.data import get_dataset + + ds = get_dataset(args.name, **_overrides(args)) + for s in ds[: args.n]: + print(json.dumps(s.model_dump(), ensure_ascii=False, indent=2)) + print('---') + return 0 + + +def _cmd_data_unload(args) -> int: + from evalharness.data import get_dataset + + for name in args.names: + ds = get_dataset(name, **_overrides(args)) + removed = ds.unload() + print(f'{name}: cache {"removed" if removed else "not present (nothing to do)"} -> {ds.cache_dir}') + return 0 + + +def _add_override_flags(p: argparse.ArgumentParser) -> None: + p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)') + p.add_argument('--split', help='override DatasetSpec.split') + p.add_argument('--subset', help='override DatasetSpec.subset') + p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)') + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI') + sub = parser.add_subparsers(dest='command', required=True) + + data = sub.add_parser('data', help='dataset plugin commands') + dsub = data.add_subparsers(dest='data_command', required=True) + + p = dsub.add_parser('list', help='list registered datasets (no download)') + p.set_defaults(func=_cmd_data_list) + + p = dsub.add_parser('fetch', help='materialize dataset(s) into the cache') + p.add_argument('names', nargs='+') + p.add_argument('--force', action='store_true', help='re-download and rebuild the cache') + p.add_argument('--workers', type=int, default=8, help='concurrent downloads (default 8)') + _add_override_flags(p) + p.set_defaults(func=_cmd_data_fetch) + + p = dsub.add_parser('unload', help='drop cache entries (raw + samples); images belong to the sandbox layer') + p.add_argument('names', nargs='+') + _add_override_flags(p) + p.set_defaults(func=_cmd_data_unload) + + p = dsub.add_parser('stats', help='materialize and show dataset statistics') + p.add_argument('name') + _add_override_flags(p) + p.set_defaults(func=_cmd_data_stats) + + p = dsub.add_parser('show', help='print the first N samples') + p.add_argument('name') + p.add_argument('-n', type=int, default=2) + _add_override_flags(p) + p.set_defaults(func=_cmd_data_show) + + return parser + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/evalharness/data/__init__.py b/evalharness/data/__init__.py new file mode 100644 index 0000000..bf8f786 --- /dev/null +++ b/evalharness/data/__init__.py @@ -0,0 +1,51 @@ +"""evalharness.data -- the data layer. + +Usage: + from evalharness.data import get_dataset, list_datasets + + ds = get_dataset('gsm8k') # lazy handle, nothing downloaded + for s in ds: # first use triggers materialize (download->convert->cache) + ... +""" + +import importlib +import pkgutil +from pathlib import Path +from typing import List + +from .dataset import Dataset +from .registry import DATASET_REGISTRY, DatasetProvider, get_dataset_provider, register_dataset +from .sample import ChatMessage, Sample, SandboxSpec, ToolInfo +from .spec import DatasetSpec, FieldSpec + +__all__ = [ + 'Dataset', 'DatasetSpec', 'FieldSpec', 'Sample', 'ChatMessage', 'SandboxSpec', 'ToolInfo', + 'register_dataset', 'get_dataset', 'list_datasets', 'get_dataset_provider', +] + + +def _discover_builtin_datasets() -> None: + """Import every plugin module under ./datasets (import = register).""" + pkg_dir = Path(__file__).parent / 'datasets' + if not pkg_dir.exists(): + return + for info in pkgutil.iter_modules([str(pkg_dir)]): + importlib.import_module(f'{__name__}.datasets.{info.name}') + + +_discover_builtin_datasets() + + +def get_dataset(name: str, **overrides) -> Dataset: + """Return a lazy Dataset handle by name. No download happens here.""" + provider = get_dataset_provider(name) + spec = provider.spec + if overrides: + import dataclasses + + spec = dataclasses.replace(spec, **overrides) + return Dataset(spec, provider.resolve_record_fn()) + + +def list_datasets() -> List[DatasetSpec]: + return [DATASET_REGISTRY.get(n).spec for n in DATASET_REGISTRY.names()] diff --git a/evalharness/data/dataset.py b/evalharness/data/dataset.py new file mode 100644 index 0000000..f4aaf79 --- /dev/null +++ b/evalharness/data/dataset.py @@ -0,0 +1,239 @@ +"""Dataset: a lazy handle over a registered dataset. + +``get_dataset('gsm8k')`` costs nothing -- no download, no parsing. The first +actual use (iteration / len / indexing) triggers ``materialize()``: + + cache hit -> read the cached samples.jsonl + cache miss -> download raw -> record_to_sample -> atomic cache write -> read + + Cache directory name: ``{safe_name}-{md5(source+split+subset+version+params)}`` + so any config change yields a different cache entry (zero invalidation logic). + + Each cache entry is self-contained: + raw/ native source data, exactly as downloaded (never converted) + samples.jsonl the unified Sample stream converted from raw/ + meta.json spec + provenance + """ + +import fcntl +import hashlib +import json +import os +import re +import string +import time +from pathlib import Path +from typing import Any, Callable, Dict, Iterator, List, Optional, Union + +from .loader import load_raw_records +from .sample import Sample +from .spec import DatasetSpec + +CACHE_ROOT = Path(os.environ.get('EVALHARNESS_CACHE', '~/.cache/evalharness')).expanduser() / 'datasets' + + +def safe_filename(s: str, max_length: int = 255) -> str: + safe_chars = string.ascii_letters + string.digits + '.-_' + s = ''.join(c if c in safe_chars else '_' for c in s) + s = re.sub(r'_+', '_', s).strip('._') + return (s or 'untitled')[:max_length] + + +def gen_hash(s: str) -> str: + return hashlib.md5(s.encode('utf-8')).hexdigest() + + +def set_cache_root(path) -> None: + """Override the cache root at runtime (used by the CLI --cache-dir flag).""" + global CACHE_ROOT + CACHE_ROOT = Path(path).expanduser() / 'datasets' + + +class Dataset: + """Sequence-like lazy dataset. Nothing is downloaded until first use.""" + + def __init__(self, spec: DatasetSpec, record_fn: Callable, samples: Optional[List[Sample]] = None): + self.spec = spec + self._record_fn = record_fn + self._samples = samples # None => not materialized yet + self.lineage: Dict[str, Any] = {} + + # ---------------- materialization ---------------- + + @property + def cache_dir(self) -> Path: + key = f'{self.spec.source}{self.spec.split}{self.spec.subset}{self.spec.version}{self.spec.params}' + # Layout: datasets//_[-]-/ + # Readable benchmark folder + readable subset/split; the 6-char hash + # suffix disambiguates different sources/params that would otherwise + # collide on the same subset_split name (correctness requirement). + parts = f'{safe_filename(self.spec.subset)}_{safe_filename(self.spec.split)}' + if self.spec.version: + parts += f'_{safe_filename(self.spec.version)}' + subdir = f'{parts}-{gen_hash(key)[:6]}' + return CACHE_ROOT / safe_filename(self.spec.name) / subdir + + @property + def is_materialized(self) -> bool: + return self._samples is not None + + def materialize(self, force: bool = False) -> 'Dataset': + if self._samples is not None and not force: + return self + cache_dir = self.cache_dir + cache_file = cache_dir / 'samples.jsonl' + if cache_file.exists() and not force: + self._samples = self._read_cache(cache_file) + self.lineage = {'from': 'cache', 'cache_dir': str(cache_dir)} + return self + + cache_dir.parent.mkdir(parents=True, exist_ok=True) + lock_path = cache_dir.with_suffix('.lock') + with open(lock_path, 'w') as lock_f: # noqa: PTH123 + fcntl.flock(lock_f, fcntl.LOCK_EX) + try: + if cache_file.exists() and not force: # double-check under lock + self._samples = self._read_cache(cache_file) + self.lineage = {'from': 'cache', 'cache_dir': str(cache_dir)} + return self + cache_dir.parent.mkdir(parents=True, exist_ok=True) + tmp_dir = cache_dir.with_name(cache_dir.name + f'.tmp-{os.getpid()}') + if tmp_dir.exists(): + import shutil + + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + records = load_raw_records(self.spec, raw_dir=tmp_dir / 'raw') + samples = [self._to_sample(r) for r in records] + self._assign_ids(samples) + self._write_cache_atomic(samples, cache_dir, tmp_dir) + self._samples = samples + self.lineage = {'from': 'source', 'cache_dir': str(cache_dir)} + finally: + fcntl.flock(lock_f, fcntl.LOCK_UN) + return self + + def _to_sample(self, record: Dict[str, Any]) -> Sample: + sample = self._record_fn(record) + if not sample.task_type: + sample.task_type = self.spec.task_type + return sample + + @staticmethod + def _assign_ids(samples: List[Sample]) -> None: + """Assign sequential ids to samples that don't carry one.""" + for i, s in enumerate(samples): + if s.id is None: + s.id = i + + def _read_cache(self, cache_file: Path) -> List[Sample]: + with open(cache_file, encoding='utf-8') as f: + samples = [Sample.model_validate(json.loads(line)) for line in f if line.strip()] + return samples + + def _write_cache_atomic(self, samples: List[Sample], cache_dir: Path, tmp_dir: Path) -> None: + """Finalize the tmp dir (samples + meta next to raw/) and swap it in. + + ``raw/`` was already populated by ``load_raw_records`` inside tmp_dir; + the tmp+rename swap makes the whole entry (raw + samples + meta) + appear atomically. + """ + with open(tmp_dir / 'samples.jsonl', 'w', encoding='utf-8') as f: + for s in samples: + f.write(json.dumps(s.model_dump(), ensure_ascii=False) + '\n') + raw_dir = tmp_dir / 'raw' + meta = { + 'spec': {k: v for k, v in vars(self.spec).items()}, + 'num_samples': len(samples), + 'raw_files': sorted(p.name for p in raw_dir.iterdir()) if raw_dir.exists() else [], + 'note': 'raw/ holds the native source data exactly as downloaded; samples.jsonl is the converted view', + 'created_at': time.strftime('%Y-%m-%d %H:%M:%S'), + } + with open(tmp_dir / 'meta.json', 'w', encoding='utf-8') as f: + json.dump(meta, f, ensure_ascii=False, indent=2) + final = cache_dir + if final.exists(): + import shutil + + # retire the old entry, then swap the new one in + old = cache_dir.with_name(cache_dir.name + f'.old-{os.getpid()}') + os.rename(final, old) + try: + os.rename(tmp_dir, final) + except OSError: + os.rename(old, final) + raise + shutil.rmtree(old, ignore_errors=True) + else: + os.rename(tmp_dir, final) + + def unload(self) -> bool: + """Drop this dataset's cache entry (raw/ + samples.jsonl + meta.json). + + Pure cache management: in-memory samples (if any) stay usable; the + next materialize rebuilds from source. Execution-environment + resources (docker images declared by ``Sample.sandbox``) are NOT + touched -- those belong to the sandbox layer's lifecycle. + """ + import shutil + + removed = False + if self.cache_dir.exists(): + shutil.rmtree(self.cache_dir) + removed = True + lock_path = self.cache_dir.with_suffix('.lock') + if lock_path.exists(): + lock_path.unlink() + if self._samples is not None or self.lineage: + self._samples = None + self.lineage = {} + return removed + + # ---------------- sequence protocol (triggers materialize) ---------------- + + def _require(self) -> List[Sample]: + self.materialize() + return self._samples + + def __iter__(self) -> Iterator[Sample]: + return iter(self._require()) + + def __len__(self) -> int: + return len(self._require()) + + def __getitem__(self, i: Union[int, slice]) -> Union[Sample, List[Sample]]: + return self._require()[i] + + # ---------------- views / derived data ---------------- + + def view(self, samples: List[Sample], lineage: Optional[Dict[str, Any]] = None) -> 'Dataset': + """An in-memory derived dataset (filter/sample/dedup results). + + Same class, same interface; ``lineage`` records how it was produced. + """ + derived = Dataset(self.spec, self._record_fn, samples=samples) + derived.lineage = {'from': 'derived', 'parent': self.spec.name, **(lineage or {})} + return derived + + # ---------------- introspection ---------------- + + def stats(self) -> Dict[str, Any]: + samples = self._require() + lengths = [len(s.input_text) for s in samples] + targets = [s.target if isinstance(s.target, str) else ','.join(s.target) for s in samples] + return { + 'name': self.spec.name, + 'task_type': self.spec.task_type, + 'num_samples': len(samples), + 'input_len': { + 'min': min(lengths) if lengths else 0, + 'max': max(lengths) if lengths else 0, + 'mean': round(sum(lengths) / len(lengths), 1) if lengths else 0, + }, + 'target_top': sorted({t: targets.count(t) for t in set(targets)}.items(), key=lambda kv: -kv[1])[:10], + 'cache_dir': str(self.cache_dir), + } + + def __repr__(self) -> str: + state = 'materialized' if self.is_materialized else 'lazy' + return f"Dataset(name={self.spec.name!r}, type={self.spec.task_type!r}, {state})" diff --git a/evalharness/data/datasets/__init__.py b/evalharness/data/datasets/__init__.py new file mode 100644 index 0000000..4b53d7a --- /dev/null +++ b/evalharness/data/datasets/__init__.py @@ -0,0 +1,6 @@ +"""Built-in dataset plugins. + +Each subdirectory is a self-contained plugin: a ``plugin.py`` (registration) +plus its data files. Subpackages are imported lazily by +``evalharness.data._discover_builtin_datasets`` via their __init__.py. +""" diff --git a/evalharness/data/datasets/aime24.py b/evalharness/data/datasets/aime24.py new file mode 100644 index 0000000..5980a72 --- /dev/null +++ b/evalharness/data/datasets/aime24.py @@ -0,0 +1,29 @@ +"""AIME 2024 (community-standard mirror: HuggingFaceH4/aime_2024). + +AIME has no official HF release; this is the widely used mirror. +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='aime24', + source='HuggingFaceH4/aime_2024', # https://huggingface.co/datasets/HuggingFaceH4/aime_2024 + split='train', # the dataset ships a single split + task_type='math', + tags=['math', 'competition'], + description='AIME 2024, 30 problems (integer answers 000-999).', + ) +) +def aime24(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['problem'], + target=str(record['answer']).strip(), + metadata={'id': record.get('id'), 'solution': record.get('solution')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/aime25.py b/evalharness/data/datasets/aime25.py new file mode 100644 index 0000000..971a3e7 --- /dev/null +++ b/evalharness/data/datasets/aime25.py @@ -0,0 +1,26 @@ +"""AIME 2025 (community-standard mirror: yentinglin/aime_2025).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='aime25', + source='yentinglin/aime_2025', # https://huggingface.co/datasets/yentinglin/aime_2025 + split='train', + task_type='math', + tags=['math', 'competition'], + description='AIME 2025, 30 problems (integer answers 000-999).', + ) +) +def aime25(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['problem'], + target=str(record['answer']).strip(), + metadata={'id': record.get('id')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/aime26.py b/evalharness/data/datasets/aime26.py new file mode 100644 index 0000000..6ad41ed --- /dev/null +++ b/evalharness/data/datasets/aime26.py @@ -0,0 +1,27 @@ +"""AIME 2026. No official standalone release; community-curated (evalscope).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='aime26', + source='evalscope/aime26', # curated; no official upstream release (ModelScope) + split='test', + task_type='math', + tags=['math', 'competition'], + description='AIME 2026 (community-curated, no official upstream).', + params={'hub': 'modelscope', 'ms_files': ['aime2026.jsonl']}, + ) +) +def aime26(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['problem'], + target=str(record['answer']).strip(), + metadata={'id': record.get('id')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/arc.py b/evalharness/data/datasets/arc.py new file mode 100644 index 0000000..0dad9fe --- /dev/null +++ b/evalharness/data/datasets/arc.py @@ -0,0 +1,29 @@ +"""AI2 ARC (official source: allenai/ai2_arc, ARC-Easy / ARC-Challenge).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='arc', + source='allenai/ai2_arc', # official: https://huggingface.co/datasets/allenai/ai2_arc + subset='ARC-Easy', # or ARC-Challenge + split='test', + task_type='mcq', + tags=['knowledge', 'science'], + description='AI2 Reasoning Challenge (official).', + ) +) +def arc(): + def to_sample(record: dict) -> Sample: + choices = record['choices'] # {'text': [...], 'label': [...]} + return Sample( + input=record['question'], + choices=list(choices['text']), + target=str(record['answerKey']).strip(), # letter label + metadata={'id': record.get('id'), 'labels': list(choices['label'])}, + ) + + return to_sample diff --git a/evalharness/data/datasets/bbh.py b/evalharness/data/datasets/bbh.py new file mode 100644 index 0000000..4ac96fa --- /dev/null +++ b/evalharness/data/datasets/bbh.py @@ -0,0 +1,23 @@ +"""BIG-Bench Hard (standard mirror: lukaemon/bbh; original: github.com/suzgunmirac/BIG-Bench-Hard).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='bbh', + source='lukaemon/bbh', # https://huggingface.co/datasets/lukaemon/bbh + subset='boolean_expressions', # 27 subtasks; override with --subset + split='test', + task_type='qa', + tags=['reasoning'], + description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/).', + ) +) +def bbh(): + def to_sample(record: dict) -> Sample: + return Sample(input=record['input'], target=str(record['target']).strip()) + + return to_sample diff --git a/evalharness/data/datasets/bfcl_v3.py b/evalharness/data/datasets/bfcl_v3.py new file mode 100644 index 0000000..ac9607f --- /dev/null +++ b/evalharness/data/datasets/bfcl_v3.py @@ -0,0 +1,63 @@ +"""BFCL v3 (Berkeley Function Calling Leaderboard). + +Official release: github.com/gorilla-llm/Berkeley-Function-Calling-Leaderboard. +We load the ModelScope mirror (AI-ModelScope/bfcl_v3, parquet) with columns +``id / turns / tools / test_category / ground_truth``. Filter by +``metadata.test_category`` at eval time (simple / irrelevance / multi_turn / +parallel / java / javascript / ...). +""" + +import json + +from ..sample import ChatMessage, Sample, ToolInfo +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='bfcl_v3', + source='AI-ModelScope/bfcl_v3', # ModelScope mirror of the official GitHub data + split='train', # the mirror ships a single split + task_type='fc', + tags=['function_calling', 'tool_use'], + description='BFCL v3 function calling (official content, ModelScope mirror).', + params={'hub': 'modelscope'}, + ) +) +def bfcl_v3(): + def _parse(v): + if isinstance(v, str): + try: + return json.loads(v) + except (ValueError, TypeError): + return v + return v + + def to_sample(record: dict) -> Sample: + messages = [] + for turn in _parse(record.get('turns')) or []: + messages.extend(turn if isinstance(turn, list) else [turn]) + tools = [] + for t in _parse(record.get('tools')) or []: + spec = t.get('function') if isinstance(t, dict) and 'function' in t else t + if isinstance(spec, dict) and spec.get('name'): + tools.append(ToolInfo(name=spec['name'], description=spec.get('description'), + parameters=spec.get('parameters') or {})) + return Sample( + input=[ChatMessage(role=m.get('role', 'user'), content=m['content'] if isinstance(m.get('content'), str) + else json.dumps(m['content'], ensure_ascii=False)) + for m in messages if isinstance(m, dict)] or record.get('id', ''), + target=json.dumps(_parse(record.get('ground_truth')), ensure_ascii=False) + if record.get('ground_truth') is not None else '', + tools=tools or None, + metadata={ + 'id': record.get('id'), + 'test_category': record.get('test_category'), + 'multi_turn': record.get('multi_turn'), + 'language': record.get('language'), + 'functions': _parse(record.get('functions')), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/bigcodebench.py b/evalharness/data/datasets/bigcodebench.py new file mode 100644 index 0000000..9debc28 --- /dev/null +++ b/evalharness/data/datasets/bigcodebench.py @@ -0,0 +1,32 @@ +"""BigCodeBench (official source: bigcode/bigcodebench).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='bigcodebench', + source='bigcode/bigcodebench', # official: https://huggingface.co/datasets/bigcode/bigcodebench + split='v0.1.4', # BigCodeBench versions are published as splits + task_type='coding', + tags=['code'], + description='BigCodeBench: practical library-level function synthesis (official).', + ) +) +def bigcodebench(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['instruct_prompt'], # 'complete_prompt' is the alternative prompt style + target=record['canonical_solution'], + metadata={ + 'task_id': record['task_id'], + 'test': record['test'], + 'entry_point': record['entry_point'], + 'code_prompt': record.get('code_prompt'), + 'libs': record.get('libs'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/cmmlu.py b/evalharness/data/datasets/cmmlu.py new file mode 100644 index 0000000..68f788e --- /dev/null +++ b/evalharness/data/datasets/cmmlu.py @@ -0,0 +1,50 @@ +"""CMMLU dataset plugin (mirror: evalscope/cmmlu on ModelScope). + +The mirror packs all 67 subjects into one parquet with a ``category`` +column, so the loader filters by ``subset`` (filter_column convention). +The official HF repo (haonan-li/cmmlu) is script-based; use --subset to +pick a subject, 'all' for everything. +""" + +import re + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='cmmlu', + source='evalscope/cmmlu', # ModelScope parquet mirror; the HF original is script-based + subset='anatomy', # 67 subjects; override with --subset or 'all' + split='test', + task_type='mcq', + tags=['zh', 'knowledge'], + description='Chinese multiple-choice QA (official content, ModelScope mirror).', + params={'hub': 'modelscope', 'filter_column': 'category'}, + ) +) +def cmmlu(): + def to_sample(record: dict) -> Sample: + # mirror layout: question/choices(['(A) ...', ...])/answer('(B) ...'); official: Question/A-D/Answer + if 'Question' in record: + choices = [record[k] for k in ('A', 'B', 'C', 'D') if record.get(k) is not None] + return Sample( + input=record['Question'], + choices=choices, + target=str(record.get('Answer', '')).strip(), + metadata={'category': record.get('Subject')}, + ) + choices = [re.sub(r'^\([A-J]\)\s*', '', c) for c in record['choices']] + answer = str(record.get('answer', '')) + m = re.match(r'^\(?([A-J])\)?', answer) + target = m.group(1) if m and len(answer) > 1 else answer + return Sample( + input=record['question'], + choices=choices, + target=target, + metadata={'category': record.get('category'), 'id': record.get('id')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/competition_math.py b/evalharness/data/datasets/competition_math.py new file mode 100644 index 0000000..2600589 --- /dev/null +++ b/evalharness/data/datasets/competition_math.py @@ -0,0 +1,44 @@ +"""MATH (official source: EleutherAI/hendrycks_math, the maintained Hendrycks MATH).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +def _extract_boxed(text: str) -> str: + """Extract the last \\boxed{...} content with brace balancing.""" + idx = text.rfind('\\boxed{') + if idx < 0: + return '' + i = idx + len('\\boxed{') + depth, start = 1, i + while i < len(text) and depth: + if text[i] == '{': + depth += 1 + elif text[i] == '}': + depth -= 1 + i += 1 + return text[start : i - 1] if depth == 0 else '' + + +@register_dataset( + DatasetSpec( + name='competition_math', + source='EleutherAI/hendrycks_math', # https://huggingface.co/datasets/EleutherAI/hendrycks_math + subset='algebra', # 7 subjects; override with --subset + split='test', + task_type='math', + tags=['math'], + description='MATH competition problems (Hendrycks). Target = \\boxed answer.', + ) +) +def competition_math(): + def to_sample(record: dict) -> Sample: + solution = record['solution'] + return Sample( + input=record['problem'], + target=_extract_boxed(solution) or solution.strip(), + metadata={'level': record.get('level'), 'type': record.get('type'), 'solution': solution}, + ) + + return to_sample diff --git a/evalharness/data/datasets/drop.py b/evalharness/data/datasets/drop.py new file mode 100644 index 0000000..2a17664 --- /dev/null +++ b/evalharness/data/datasets/drop.py @@ -0,0 +1,31 @@ +"""DROP (official source: ucinlp/drop).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='drop', + source='ucinlp/drop', # official: https://huggingface.co/datasets/ucinlp/drop + split='validation', + task_type='qa', + tags=['reading_comprehension'], + description='DROP reading comprehension; target = answer spans list.', + ) +) +def drop(): + def to_sample(record: dict) -> Sample: + spans = record['answers_spans']['spans'] + return Sample( + input=record['question'], + target=list(spans) if len(spans) > 1 else spans[0], + metadata={ + 'passage': record['passage'], # required reading context, kept out of input + 'query_id': record.get('query_id'), + 'section_id': record.get('section_id'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/general_fc.py b/evalharness/data/datasets/general_fc.py new file mode 100644 index 0000000..7911040 --- /dev/null +++ b/evalharness/data/datasets/general_fc.py @@ -0,0 +1,34 @@ +"""general_fc: simple function-calling demo set (native to evalscope, its official home).""" + +import json + +from ..sample import ChatMessage, Sample, ToolInfo +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='general_fc', + source='evalscope/GeneralFunctionCall-Test', # evalscope-native release (ModelScope) + split='test', + task_type='fc', + tags=['function_calling'], + description='Minimal function-calling test set (evalscope-native).', + params={'hub': 'modelscope'}, + ) +) +def general_fc(): + def to_sample(record: dict) -> Sample: + messages = json.loads(record['messages']) if isinstance(record['messages'], str) else record['messages'] + tools = json.loads(record['tools']) if isinstance(record['tools'], str) else record.get('tools') + chat = [ChatMessage(role=m.get('role', 'user'), content=m['content'] if isinstance(m.get('content'), str) + else json.dumps(m['content'], ensure_ascii=False)) for m in messages] + return Sample( + input=chat, + target=str(record.get('should_call_tool', '')), + tools=[ToolInfo(**t['function']) if isinstance(t, dict) and 'function' in t else ToolInfo(name=str(t)) + for t in (tools or [])] or None, + ) + + return to_sample diff --git a/evalharness/data/datasets/gpqa_diamond.py b/evalharness/data/datasets/gpqa_diamond.py new file mode 100644 index 0000000..a7fa9d2 --- /dev/null +++ b/evalharness/data/datasets/gpqa_diamond.py @@ -0,0 +1,43 @@ +"""GPQA Diamond. + +Official source Idavidrein/gpqa is gated on HF (needs auth); we load the +ModelScope mirror (AI-ModelScope/gpqa_diamond) which keeps the official +column layout (Question / Correct Answer / Incorrect Answer 1-3). + +Note: choices are stored with the correct answer first (target 'A'). +Option shuffling is an eval-time concern (the evaluator should shuffle +choices and remap the target, like Dataset.shuffle_choices would). +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='gpqa_diamond', + source='AI-ModelScope/gpqa_diamond', # ModelScope mirror of the gated official + split='train', + task_type='mcq', + tags=['knowledge', 'science'], + description='GPQA diamond split, graduate-level science MCQ (official content).', + params={'hub': 'modelscope'}, + ) +) +def gpqa_diamond(): + def to_sample(record: dict) -> Sample: + choices = [ + record['Correct Answer'], + record['Incorrect Answer 1'], + record['Incorrect Answer 2'], + record['Incorrect Answer 3'], + ] + return Sample( + input=record['Question'], + choices=choices, + target='A', # correct answer is first; shuffle at eval time + metadata={'subdomain': record.get('Subdomain'), 'unshuffled': True}, + ) + + return to_sample diff --git a/evalharness/data/datasets/gsm8k.py b/evalharness/data/datasets/gsm8k.py new file mode 100644 index 0000000..1ae3845 --- /dev/null +++ b/evalharness/data/datasets/gsm8k.py @@ -0,0 +1,33 @@ +"""GSM8K dataset plugin (official source: openai/gsm8k). + +Offline demo: examples/data/gsm8k_main_test.jsonl ships a tiny subset, e.g. +``evalharness data fetch gsm8k --source examples/data/gsm8k_main_test.jsonl`` +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='gsm8k', + source='openai/gsm8k', # official: https://huggingface.co/datasets/openai/gsm8k + subset='main', + split='test', + task_type='math', + tags=['math', 'cot'], + description='Grade school math word problems (OpenAI, official).', + ) +) +def gsm8k(): + def to_sample(record: dict) -> Sample: + parts = record['answer'].split('####') + target = parts.pop().strip() + return Sample( + input=record['question'], + target=target, + metadata={'reasoning': '####'.join(parts).strip()}, + ) + + return to_sample diff --git a/evalharness/data/datasets/hellaswag.py b/evalharness/data/datasets/hellaswag.py new file mode 100644 index 0000000..0d720ca --- /dev/null +++ b/evalharness/data/datasets/hellaswag.py @@ -0,0 +1,29 @@ +"""HellaSwag (official source: rowanz/hellaswag).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + +_LETTERS = 'ABCD' + + +@register_dataset( + DatasetSpec( + name='hellaswag', + source='Rowan/hellaswag', # parquet conversion of the official rowanz/hellaswag + split='validation', + task_type='mcq', + tags=['commonsense'], + description='HellaSwag commonsense sentence completion (official).', + ) +) +def hellaswag(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['ctx'], + choices=list(record['endings']), + target=_LETTERS[int(record['label'])], + metadata={'activity_label': record.get('activity_label')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/hle.py b/evalharness/data/datasets/hle.py new file mode 100644 index 0000000..80f2742 --- /dev/null +++ b/evalharness/data/datasets/hle.py @@ -0,0 +1,37 @@ +"""HLE - Humanity's Last Exam. + +Official source cais/hle is gated on HF (needs auth); we load the ModelScope +mirror of the identical data via the native raw-file downloader. +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='hle', + source='cais/hle', # ModelScope mirror of the gated HF original + split='test', + task_type='qa', + tags=['knowledge', 'frontier'], + description="Humanity's Last Exam. Some samples carry an image field (multimodal).", + params={'hub': 'modelscope'}, + ) +) +def hle(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['question'], + target=str(record['answer']).strip(), + metadata={ + 'id': record.get('id'), + 'answer_type': record.get('answer_type'), + 'category': record.get('category'), + 'raw_subject': record.get('raw_subject'), + 'has_image': bool(record.get('image')), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/hmmt26.py b/evalharness/data/datasets/hmmt26.py new file mode 100644 index 0000000..022ac4a --- /dev/null +++ b/evalharness/data/datasets/hmmt26.py @@ -0,0 +1,27 @@ +"""HMMT Feb 2026. No official standalone release; community-curated (evalscope).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='hmmt26', + source='evalscope/hmmt_feb_2026', # curated; no official upstream release (ModelScope) + split='train', # the mirror ships a single train split + task_type='math', + tags=['math', 'competition'], + description='HMMT February 2026 (community-curated, no official upstream).', + params={'hub': 'modelscope'}, + ) +) +def hmmt26(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['problem'], + target=str(record['answer']).strip(), + metadata={'problem_idx': record.get('problem_idx'), 'problem_type': record.get('problem_type')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/humaneval.py b/evalharness/data/datasets/humaneval.py new file mode 100644 index 0000000..09efec9 --- /dev/null +++ b/evalharness/data/datasets/humaneval.py @@ -0,0 +1,30 @@ +"""HumanEval (official source: openai/openai_humaneval).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='humaneval', + source='openai/openai_humaneval', # official: https://huggingface.co/datasets/openai/openai_humaneval + split='test', + task_type='coding', + tags=['code'], + description='OpenAI HumanEval function synthesis (official). Tests in metadata for sandbox.', + ) +) +def humaneval(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['prompt'], + target=record['canonical_solution'], + metadata={ + 'task_id': record['task_id'], + 'test': record['test'], + 'entry_point': record['entry_point'], + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/imo_answerbench.py b/evalharness/data/datasets/imo_answerbench.py new file mode 100644 index 0000000..b01e50a --- /dev/null +++ b/evalharness/data/datasets/imo_answerbench.py @@ -0,0 +1,32 @@ +"""IMO Answer Bench. Community-curated (evalscope); no official upstream release.""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='imo_answerbench', + source='evalscope/imo-answerbench', # curated; no official upstream release (ModelScope) + split='train', # the mirror ships a single train split + task_type='math', + tags=['math', 'competition', 'imo'], + description='IMO-level answer bench (community-curated, no official upstream).', + params={'hub': 'modelscope'}, + ) +) +def imo_answerbench(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['Problem'], + target=str(record['Short Answer']).strip(), + metadata={ + 'id': record.get('Problem ID'), + 'category': record.get('Category'), + 'subcategory': record.get('Subcategory'), + 'source': record.get('Source'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/live_code_bench.py b/evalharness/data/datasets/live_code_bench.py new file mode 100644 index 0000000..11ab3b9 --- /dev/null +++ b/evalharness/data/datasets/live_code_bench.py @@ -0,0 +1,38 @@ +"""LiveCodeBench code generation lite (official source: livecodebench/code_generation_lite).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='live_code_bench', + source='evalscope/livecodebench_code_generation_lite_parquet', # parquet mirror (ModelScope); + # the HF original is script-based and no longer loadable by datasets>=5 + subset='release_latest', # or release_v1..v6 / vX / vX_vY version windows + split='test', + task_type='coding', + tags=['code'], + description='LiveCodeBench (lite) contest problems; version tags via DatasetSpec.version.', + params={'hub': 'modelscope'}, + ) +) +def live_code_bench(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['question_content'], + target='', # judged by running hidden test cases, no reference text + metadata={ + 'question_id': record['question_id'], + 'contest_id': record.get('contest_id'), + 'contest_date': record.get('contest_date'), + 'platform': record.get('platform'), + 'difficulty': record.get('difficulty'), + 'starter_code': record.get('starter_code'), + 'public_test_cases': record.get('public_test_cases'), + 'private_test_cases': record.get('private_test_cases'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/longbench_v2.py b/evalharness/data/datasets/longbench_v2.py new file mode 100644 index 0000000..27c7c1b --- /dev/null +++ b/evalharness/data/datasets/longbench_v2.py @@ -0,0 +1,33 @@ +"""LongBench v2 (official source: THUDM/LongBench-v2).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='longbench_v2', + source='THUDM/LongBench-v2', # official: https://huggingface.co/datasets/THUDM/LongBench-v2 + split='train', # the dataset ships a single split + task_type='mcq', + tags=['long_context'], + description='LongBench v2 long-context MCQ (official). Context kept in metadata.', + ) +) +def longbench_v2(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['question'], + choices=[record['choice_A'], record['choice_B'], record['choice_C'], record['choice_D']], + target=str(record['answer']).strip(), + metadata={ + 'context': record['context'], # the long document; eval-time prompt assembly + 'domain': record.get('domain'), + 'sub_domain': record.get('sub_domain'), + 'difficulty': record.get('difficulty'), + 'length': record.get('length'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/mmlu.py b/evalharness/data/datasets/mmlu.py new file mode 100644 index 0000000..7ce63f1 --- /dev/null +++ b/evalharness/data/datasets/mmlu.py @@ -0,0 +1,30 @@ +"""MMLU (official source: cais/mmlu).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + +_LETTERS = 'ABCDEFGHIJ' + + +@register_dataset( + DatasetSpec( + name='mmlu', + source='cais/mmlu', # official: https://huggingface.co/datasets/cais/mmlu + subset='all', # 57 subjects; override with --subset + split='test', + task_type='mcq', + tags=['knowledge'], + description='Massive Multitask Language Understanding (official).', + ) +) +def mmlu(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['question'], + choices=list(record['choices']), + target=_LETTERS[int(record['answer'])], + metadata={'subject': record.get('subject')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/mmlu_pro.py b/evalharness/data/datasets/mmlu_pro.py new file mode 100644 index 0000000..8fe2afd --- /dev/null +++ b/evalharness/data/datasets/mmlu_pro.py @@ -0,0 +1,27 @@ +"""MMLU-Pro (official source: TIGER-Lab/MMLU-Pro).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='mmlu_pro', + source='TIGER-Lab/MMLU-Pro', # official: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro + split='test', + task_type='mcq', + tags=['knowledge'], + description='MMLU-Pro: 10-option harder MMLU (official).', + ) +) +def mmlu_pro(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['question'], + choices=list(record['options']), + target=str(record['answer']).strip(), # already a letter + metadata={'category': record.get('category'), 'question_id': record.get('question_id')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/openai_mrcr.py b/evalharness/data/datasets/openai_mrcr.py new file mode 100644 index 0000000..1be7ba8 --- /dev/null +++ b/evalharness/data/datasets/openai_mrcr.py @@ -0,0 +1,32 @@ +"""OpenAI MRCR (multi-round coreference, long-context). Mirror: openai-mirror/mrcr.""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='openai_mrcr', + source='openai-mirror/mrcr', # OpenAI MRCR mirror on ModelScope + subset='2needle', # or 4needle / 8needle + split='test', + task_type='qa', + tags=['long_context'], + description='OpenAI MRCR long-context retrieval/coreference (community mirror).', + params={'hub': 'modelscope'}, + ) +) +def openai_mrcr(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['prompt'], + target=str(record['answer']).strip(), + metadata={ + 'n_needles': record.get('n_needles'), + 'total_messages': record.get('total_messages'), + 'random_string_to_prepend': record.get('random_string_to_prepend'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/simple_qa.py b/evalharness/data/datasets/simple_qa.py new file mode 100644 index 0000000..d3e7ebc --- /dev/null +++ b/evalharness/data/datasets/simple_qa.py @@ -0,0 +1,29 @@ +"""SimpleQA (OpenAI; official release is the CSV in github.com/openai/simple-evals). + +The HF id below is the standard community mirror of that CSV. +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='simple_qa', + source='basicv8vc/SimpleQA', # mirror of the official openai/simple-evals CSV + split='test', + task_type='qa', + tags=['factuality'], + description='SimpleQA factuality benchmark (OpenAI, community mirror of official CSV).', + ) +) +def simple_qa(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['problem'], + target=str(record['answer']).strip(), + metadata={'metadata': record.get('metadata')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/swe_bench_verified.py b/evalharness/data/datasets/swe_bench_verified.py new file mode 100644 index 0000000..5415fd7 --- /dev/null +++ b/evalharness/data/datasets/swe_bench_verified.py @@ -0,0 +1,38 @@ +"""SWE-bench Verified (official source: princeton-nlp/SWE-bench_Verified).""" + +from ..sample import Sample, SandboxSpec +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='swe_bench_verified', + source='princeton-nlp/SWE-bench_Verified', # official: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified + split='test', + task_type='agent', + tags=['code', 'agent', 'swe'], + requires=['docker'], + description='SWE-bench Verified; per-instance docker image carried in Sample.sandbox.', + ) +) +def swe_bench_verified(): + def to_sample(record: dict) -> Sample: + instance_id = record['instance_id'] + return Sample( + input=record['problem_statement'], + target=record['patch'], # gold patch (for oracle/oracle-check only) + sandbox=SandboxSpec(image=f'sweb.eval.x86_64.{instance_id}'), # official image naming + metadata={ + 'instance_id': instance_id, + 'repo': record['repo'], + 'base_commit': record['base_commit'], + 'test_patch': record['test_patch'], + 'FAIL_TO_PASS': record['FAIL_TO_PASS'], + 'PASS_TO_PASS': record['PASS_TO_PASS'], + 'environment_setup_commit': record.get('environment_setup_commit'), + 'difficulty': record.get('difficulty'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/tau2_bench.py b/evalharness/data/datasets/tau2_bench.py new file mode 100644 index 0000000..5e94084 --- /dev/null +++ b/evalharness/data/datasets/tau2_bench.py @@ -0,0 +1,46 @@ +"""tau2-bench (Sierra). Official release: github.com/sierra-research/tau2-bench. + +We load the ModelScope mirror of the official task files +(evalscope/tau2-bench-data, same repo layout: tau2/domains//tasks.json). +Each record is a full task: agent purpose + user scenario + evaluation criteria. +""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + +_DOMAINS = ('airline', 'retail', 'telecom', 'mock') + + +@register_dataset( + DatasetSpec( + name='tau2_bench', + source='evalscope/tau2-bench-data', # mirror of the official GitHub data + subset='airline', # or retail / telecom / mock; override with --subset + split='test', + task_type='agent', + tags=['agent', 'tool_use', 'dialog'], + description='tau2-bench agent-tool-dialog tasks (official content, ModelScope mirror).', + params={'hub': 'modelscope', 'ms_files': ['tau2/domains/{subset}/tasks.json']}, + ) +) +def tau2_bench(): + def to_sample(record: dict) -> Sample: + desc = record.get('description') or {} + scenario = record.get('user_scenario') or {} + instructions = (scenario.get('instructions') or {}).get('task_instructions') + return Sample( + input=desc.get('purpose') or record.get('id', ''), + target='', + metadata={ + 'id': record.get('id'), + 'notes': desc.get('notes'), + 'task_instructions': instructions, + 'user_scenario': scenario, + 'initial_state': record.get('initial_state'), + 'evaluation_criteria': record.get('evaluation_criteria'), + 'annotations': record.get('annotations'), + }, + ) + + return to_sample diff --git a/evalharness/data/datasets/trivia_qa.py b/evalharness/data/datasets/trivia_qa.py new file mode 100644 index 0000000..af89aba --- /dev/null +++ b/evalharness/data/datasets/trivia_qa.py @@ -0,0 +1,29 @@ +"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.nocontext config).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='trivia_qa', + source='mandarjoshi/trivia_qa', # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa + subset='rc.nocontext', + split='validation', + task_type='qa', + tags=['knowledge', 'openqa'], + description='TriviaQA open-domain QA without context (official).', + ) +) +def trivia_qa(): + def to_sample(record: dict) -> Sample: + answer = record['answer'] # {'value': ..., 'aliases': [...], ...} + targets = [answer['value']] + list(answer.get('aliases') or []) + return Sample( + input=record['question'], + target=targets, # multi-target: any alias counts + metadata={'question_id': record.get('question_id')}, + ) + + return to_sample diff --git a/evalharness/data/datasets/winogrande.py b/evalharness/data/datasets/winogrande.py new file mode 100644 index 0000000..b524015 --- /dev/null +++ b/evalharness/data/datasets/winogrande.py @@ -0,0 +1,27 @@ +"""Winogrande (official source: allenai/winogrande, winogrande_xl).""" + +from ..sample import Sample +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='winogrande', + source='allenai/winogrande', # official: https://huggingface.co/datasets/allenai/winogrande + subset='winogrande_xl', + split='validation', + task_type='mcq', + tags=['commonsense', 'coreference'], + description='Winogrande XL binary coreference (official).', + ) +) +def winogrande(): + def to_sample(record: dict) -> Sample: + return Sample( + input=record['sentence'], + choices=[record['option1'], record['option2']], + target={'1': 'A', '2': 'B'}[record['answer']], + ) + + return to_sample diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py new file mode 100644 index 0000000..67c7296 --- /dev/null +++ b/evalharness/data/loader.py @@ -0,0 +1,265 @@ +"""Raw record loading and the FieldSpec-based default conversion. + +Supported sources: +- local file: .jsonl / .json / .csv / .tsv / .parquet +- local dir: looks for ``{subset}_{split}.jsonl`` etc. (and csv/tsv/parquet) +- hub id: HuggingFace ``datasets`` (optional dependency, imported on demand) +- modelscope: ``params={'hub': 'modelscope'}`` — native ModelScope raw-file + HTTP download (no ``modelscope`` package needed, immune to its datasets + version pinning). File selection mirrors the local-dir convention and + understands HF-style shards (``test-00000-of-00001.parquet``). +""" + +import csv +import json +import os +import re +import shutil +import urllib.request +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Union + +from .sample import Sample +from .spec import DatasetSpec, FieldSpec + +_SUPPORTED_EXTS = ['.jsonl', '.json', '.csv', '.tsv', '.parquet'] +# params keys consumed by the loader itself; never forwarded to load_dataset() +_RESERVED_PARAMS = {'hub', 'ms_files', 'filter_column'} +_MS_API = 'https://www.modelscope.cn/api/v1/datasets' + + +def load_raw_records(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[Dict[str, Any]]: + """Load raw records (list of dicts) from the spec's source. + + When ``raw_dir`` is given, a copy of the *native* source data is placed + there (downloaded blob / copy of the local file / record dump for hub + sources) so every cache entry is self-contained: raw in, samples out. + """ + source = spec.source + if os.path.exists(source): + records, native_files = _load_local(source, spec) + if raw_dir is not None and native_files: + _link_or_copy_all(native_files, raw_dir) + elif spec.params.get('hub') == 'modelscope': + records = _load_from_modelscope(spec, raw_dir) + else: + records = _load_from_hub(spec) + if raw_dir is not None: + # hub-native packaging lives in the HF cache; keep an exact, + # pre-conversion record dump so the entry is self-contained + raw_dir.mkdir(parents=True, exist_ok=True) + with open(raw_dir / 'records.jsonl', 'w', encoding='utf-8') as f: + for r in records: + f.write(json.dumps(r, ensure_ascii=False, default=str) + '\n') + records = _apply_subset_filter(spec, records) + return records + + +def _load_local(path: str, spec: DatasetSpec) -> tuple: + """Load from a local path; also return the native file(s) to preserve.""" + if os.path.isfile(path): + return _read_file(path), [Path(path)] + # directory: follow the _. / . convention + for ext in _SUPPORTED_EXTS: + for candidate in (f'{spec.subset}_{spec.split}{ext}', f'{spec.subset}{ext}', f'{spec.split}{ext}'): + full = os.path.join(path, candidate) + if os.path.exists(full): + return _read_file(full), [Path(full)] + expected = [os.path.join(path, f'{spec.subset}_{spec.split}{e}') for e in _SUPPORTED_EXTS] + available = sorted(f for f in os.listdir(path) if os.path.splitext(f)[1] in _SUPPORTED_EXTS) + raise FileNotFoundError( + f'no dataset file found for subset={spec.subset!r} split={spec.split!r} in {path!r}.\n' + f'Expected one of:\n - ' + '\n - '.join(expected) + '\n' + f'Available: {available or "(none)"}' + ) + + +def _link_or_copy_all(files: List[Path], raw_dir: Path) -> None: + """Place native files in raw_dir (hardlink when possible, else copy).""" + raw_dir.mkdir(parents=True, exist_ok=True) + for src in files: + dst = raw_dir / src.name + if dst.exists() or src.parent.resolve() == raw_dir.resolve(): + continue + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + +def _read_file(path: str) -> List[Dict[str, Any]]: + ext = os.path.splitext(path)[1] + if ext == '.parquet': + try: + import pyarrow.parquet as pq + except ImportError: + raise ImportError( + f'{path} is parquet; install the reader first: pip install pyarrow' + ) + return pq.read_table(path).to_pylist() + with open(path, encoding='utf-8') as f: + if ext == '.jsonl': + return [json.loads(line) for line in f if line.strip()] + if ext == '.json': + data = json.load(f) + return data if isinstance(data, list) else [data] + if ext in ('.csv', '.tsv'): + return list(csv.DictReader(f, delimiter='\t' if ext == '.tsv' else ',')) + raise ValueError(f'unsupported file format: {path}') + + +def _apply_subset_filter(spec: DatasetSpec, records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """When ``params['filter_column']`` is set, subset selects that column's value. + + Used when a mirror packs all subsets into one file with a category column + (e.g. evalscope/cmmlu: one parquet, ``category`` = subject). + """ + col = spec.params.get('filter_column') + if not col or spec.subset in ('default', 'all'): + return records + return [r for r in records if r.get(col) == spec.subset] + + +# ---------------- ModelScope raw-file loading (no modelscope package) ---------------- + + +def _ms_api_json(url: str) -> Dict[str, Any]: + req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'}) + with urllib.request.urlopen(req, timeout=60) as resp: + return json.loads(resp.read().decode('utf-8')) + + +def _ms_list_files(repo: str, root: str = '', depth: int = 0) -> List[str]: + """Recursively list blob paths under a ModelScope dataset repo directory.""" + url = f'{_MS_API}/{repo}/repo/tree?Revision=master' + (f'&Root={root}' if root else '') + try: + data = _ms_api_json(url).get('Data') or {} + except Exception: + return [] + files: List[str] = [] + for entry in data.get('Files') or []: + if entry.get('Type') == 'blob': + files.append(entry['Path']) + elif entry.get('Type') == 'tree' and depth < 4: + files.extend(_ms_list_files(repo, entry['Path'], depth + 1)) + return files + + +def _ms_match_files(spec: DatasetSpec, files: List[str]) -> List[str]: + """Pick repo files for this subset/split, mirroring the local-dir convention. + + Priority: explicit ``params['ms_files']`` > exact name match > HF-style + shard match (``{split}-00000-of-00001`` / ``{subset}_{split}-...`` / + ``{subset}_0``) > single-data-file repos. When the subset names a parent + directory, matches inside it win (e.g. livecodebench ``release_latest/``). + """ + explicit = spec.params.get('ms_files') + if explicit: + # paths support {subset}/{split} templating (e.g. tau2 domains) + return [p.format(subset=spec.subset, split=spec.split) for p in explicit] + + def match_one(path: str) -> bool: + base = os.path.basename(path) + name = os.path.splitext(base)[0] + for ext in _SUPPORTED_EXTS: + if not base.endswith(ext): + continue + stem = base[: -len(ext)] + if stem in (f'{spec.subset}_{spec.split}', spec.subset, spec.split): + return True + if stem == f'{spec.subset}{spec.split}': + return True + shard = ( + rf'{re.escape(spec.split)}-\d+-of-\d+$' + rf'|{re.escape(spec.subset)}_{re.escape(spec.split)}[-_].*' + rf'|{re.escape(spec.subset)}[-_]\d+$' + ) + if re.fullmatch(shard, stem, flags=re.ASCII): + return True + return False + + candidates = [f for f in files if match_one(f)] + if not candidates: + data_files = [f for f in files if os.path.splitext(f)[1] in _SUPPORTED_EXTS] + if len(data_files) == 1: + return data_files + return [] + in_subset_dir = [f for f in candidates if os.path.dirname(f) == spec.subset] + return sorted(in_subset_dir or candidates) + + +def _ms_download(repo: str, path: str, dest_dir: Path) -> Path: + """Download one repo file into the raw cache (content-addressed, reused).""" + dest = dest_dir / os.path.basename(path) + if dest.exists() and dest.stat().st_size > 0: + return dest + dest_dir.mkdir(parents=True, exist_ok=True) + url = f'{_MS_API}/{repo}/repo?Revision=master&FilePath={path}' + tmp = dest.with_name(dest.name + f'.part-{os.getpid()}') + req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'}) + with urllib.request.urlopen(req, timeout=600) as resp, open(tmp, 'wb') as out: + while True: + chunk = resp.read(1 << 20) + if not chunk: + break + out.write(chunk) + os.replace(tmp, dest) + return dest + + +def _load_from_modelscope(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[Dict[str, Any]]: + import hashlib + + files = _ms_list_files(spec.source) + if not files: + raise FileNotFoundError(f'no files found on ModelScope dataset {spec.source!r}') + selected = _ms_match_files(spec, files) + if not selected: + raise FileNotFoundError( + f'no data file for subset={spec.subset!r} split={spec.split!r} in ' + f'modelscope {spec.source!r}. Available (first 10): {files[:10]}' + ) + # shared blob store: download once per repo, hardlink into each cache entry + blob_dir = Path(os.environ.get('EVALHARNESS_CACHE', '~/.cache/evalharness')).expanduser() + blob_dir = blob_dir / '.raw' / hashlib.md5(spec.source.encode()).hexdigest()[:10] + records: List[Dict[str, Any]] = [] + blobs: List[Path] = [] + for path in selected: + blobs.append(_ms_download(spec.source, path, blob_dir)) + records.extend(_read_file(str(blobs[-1]))) + if raw_dir is not None: + _link_or_copy_all(blobs, raw_dir) + return records + + +def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]: + try: + import datasets + except ImportError: + raise ImportError( + f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' + "install the optional dependency first: pip install 'evalharness[hub]'" + ) + kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} + subset = None if spec.subset == 'default' else spec.subset + ds = datasets.load_dataset(spec.source, subset, split=spec.split, revision=spec.version, **kwargs) + return [dict(r) for r in ds] + + +def field_spec_to_record_fn(fields: FieldSpec) -> Callable[[Dict[str, Any]], Sample]: + """Build the default record->Sample converter from a FieldSpec.""" + + def convert(record: Dict[str, Any]) -> Sample: + target = record.get(fields.target, '') + if isinstance(target, (int, float)): + target = str(target) + metadata = {k: record.get(k) for k in fields.metadata} + return Sample( + input=record.get(fields.input, ''), + choices=record.get(fields.choices) if fields.choices in record else None, + target=target, + id=record.get(fields.id) if fields.id else None, + metadata=metadata, + ) + + return convert diff --git a/evalharness/data/registry.py b/evalharness/data/registry.py new file mode 100644 index 0000000..ddc26e1 --- /dev/null +++ b/evalharness/data/registry.py @@ -0,0 +1,93 @@ +"""Dataset registry: decorator registration + name lookup with suggestions. + +Registration happens at import time ("import = register"). The registry maps +name -> DatasetProvider; a Dataset is only *materialized* on first use. +""" + +import difflib +from typing import Callable, Dict, List, Optional, Union + +from .spec import DatasetSpec, FieldSpec + +# A provider factory: called once, returns how records become Samples. +ProviderFactory = Callable[[], Union[FieldSpec, Callable, None]] + + +class DatasetProvider: + """A registered dataset: metadata + the recipe for converting records.""" + + def __init__(self, spec: DatasetSpec, factory: Optional[ProviderFactory] = None): + self.spec = spec + self._factory = factory + self._record_fn = None + self._resolved = False + + def resolve_record_fn(self) -> Callable: + """Resolve the record->Sample converter, lazily and exactly once.""" + if not self._resolved: + result = self._factory() if self._factory else None + if isinstance(result, FieldSpec): + from .loader import field_spec_to_record_fn + + self._record_fn = field_spec_to_record_fn(result) + elif callable(result): + self._record_fn = result + elif result is None: + self._record_fn = field_spec_to_record_fn(FieldSpec()) + else: + raise TypeError(f'{self.spec.name}: factory must return FieldSpec or callable, got {type(result)}') + self._resolved = True + return self._record_fn + + +class Registry: + """Minimal dict-like registry with duplicate protection and suggestions.""" + + def __init__(self, kind: str): + self.kind = kind + self._items: Dict[str, DatasetProvider] = {} + + def register(self, name: str, item: DatasetProvider) -> DatasetProvider: + if name in self._items: + raise ValueError(f'{self.kind} {name!r} is already registered') + self._items[name] = item + return item + + def get(self, name: str) -> DatasetProvider: + if name not in self._items: + suggestions = difflib.get_close_matches(name, self._items.keys(), n=3) + hint = f" Did you mean: {', '.join(suggestions)}?" if suggestions else '' + raise KeyError(f'unknown {self.kind} {name!r}.{hint}') + return self._items[name] + + def names(self) -> List[str]: + return sorted(self._items) + + def __contains__(self, name: str) -> bool: + return name in self._items + + def __len__(self) -> int: + return len(self._items) + + +DATASET_REGISTRY = Registry('dataset') + + +def register_dataset(spec: DatasetSpec): + """Decorator: register a dataset plugin. + + Usage: + @register_dataset(DatasetSpec(name='gsm8k', source=...)) + def gsm8k(): + return lambda record: Sample(...) # or FieldSpec(...), or None + """ + + def decorator(factory: ProviderFactory) -> ProviderFactory: + DATASET_REGISTRY.register(spec.name, DatasetProvider(spec, factory)) + return factory + + return decorator + + +def get_dataset_provider(name: str) -> DatasetProvider: + return DATASET_REGISTRY.get(name) diff --git a/evalharness/data/sample.py b/evalharness/data/sample.py new file mode 100644 index 0000000..7a7a36d --- /dev/null +++ b/evalharness/data/sample.py @@ -0,0 +1,62 @@ +"""Unified sample schema for EvalHarness. + +The data layer only *declares*; execution layers (sandbox / model / scorer) +consume these models. Raw dataset formats are free-form -- every dataset +plugin converts its records into ``Sample`` via ``record_to_sample``. +""" + +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field + + +class ChatMessage(BaseModel): + role: Literal['system', 'user', 'assistant', 'tool'] + content: str + + +class SandboxSpec(BaseModel): + """Execution environment carried by a sample (coding/agent tasks).""" + + image: Optional[str] = None + compose_file: Optional[str] = None + platform: Optional[str] = None + config: Dict[str, Any] = Field(default_factory=dict) + + +class ToolInfo(BaseModel): + """Tool declaration for function-calling / agent samples.""" + + name: str + description: Optional[str] = None + parameters: Dict[str, Any] = Field(default_factory=dict) + + +class Sample(BaseModel): + """The single currency of the data layer. + + Conventions: + - ``input`` : question text, or a list of ChatMessage for multi-turn/multimodal. + - ``choices`` : option *contents* for multiple-choice tasks. + - ``target`` : reference answer; a LETTER (e.g. 'A') for MCQ, text otherwise. + - ``id``/``group_id`` : assigned by the framework on materialize/repeats. + """ + + input: Union[str, List[ChatMessage]] + choices: Optional[List[str]] = None + target: Union[str, List[str]] = '' + id: Optional[int] = None + group_id: Optional[int] = None + task_type: Optional[str] = None # qa | mcq | math | coding | agent | vqa | ... + tools: Optional[List[ToolInfo]] = None + sandbox: Optional[SandboxSpec] = None + files: Optional[Dict[str, str]] = None # path -> content, copied into sandbox + setup: Optional[str] = None # script run in sandbox before use + metadata: Dict[str, Any] = Field(default_factory=dict) + + @property + def input_text(self) -> str: + """Unified text view of the input.""" + if isinstance(self.input, str): + return self.input + return '\n'.join(m.content for m in self.input) diff --git a/evalharness/data/spec.py b/evalharness/data/spec.py new file mode 100644 index 0000000..9037b9e --- /dev/null +++ b/evalharness/data/spec.py @@ -0,0 +1,38 @@ +"""Dataset metadata (DatasetSpec) and declarative field mapping (FieldSpec).""" + +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class FieldSpec: + """Declarative mapping: raw record field name -> Sample field name. + + Use this when the raw records are already well-shaped; no custom + ``record_to_sample`` function is needed then. + """ + + input: str = 'input' + target: str = 'target' + choices: str = 'choices' + id: Optional[str] = None + metadata: List[str] = field(default_factory=list) + + +@dataclass +class DatasetSpec: + """Everything the framework needs to know about a dataset *without* + loading it. Drives the cache key, the CLI listing, and (later) the + deployment-time dependency resolution via ``requires``. + """ + + name: str + source: str # hub id ('AI-ModelScope/gsm8k') or local path + split: str = 'test' + subset: str = 'default' + version: Optional[str] = None + task_type: str = 'qa' # qa | mcq | math | coding | agent | vqa | fc + tags: List[str] = field(default_factory=list) + requires: List[str] = field(default_factory=list) # e.g. ['docker'] + description: str = '' + params: dict = field(default_factory=dict) # extra load params, part of cache key diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a995fe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "evalharness" +version = "0.1.0" +description = "A plugin-based LLM/agent evaluation harness (data layer first)" +requires-python = ">=3.10" +dependencies = ["pydantic>=2"] + +[project.optional-dependencies] +hub = ["datasets"] # needed only for HuggingFace-hosted datasets +parquet = ["pyarrow"] # needed only for parquet sources (ModelScope mirrors) + +[project.scripts] +evalharness = "evalharness.cli:main" + +[tool.setuptools.packages.find] +include = ["evalharness*"] + +[tool.setuptools.package-data] +"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv"]