Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify install: light deps (datasets/pyarrow/sympy/pylatexenc/numpy/scipy) are DEFAULT; single [bfcl] extra for the heavy official checker

Browse Source
This commit is contained in:
sora 2026-08-25 02:51:55 +00:00
parent 99c9df5412
commit 89e721414f
4 changed files with 22 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -30,19 +30,15 @@ land next.
## Install ## Install
```bash ```bash
pip install . # core only (pydantic) pip install . # core + all light deps (datasets/pyarrow/sympy/numpy/scipy):
pip install '.[hub]' # + HuggingFace datasets # every benchmark scores officially, nothing conflicts
pip install '.[math]' # + sympy/pylatexenc (aime/math official grader) pip install '.[bfcl]' # + heavy official bfcl ast_checker (5 cloud SDKs + qwen-agent;
pip install '.[exec]' # + numpy/scipy (official DROP aligner) # only for official-parity BFCL scoring -- native default works)
pip install '.[hub,parquet,math,exec]' # everything
``` ```
For development: `pip install -e '.[hub,parquet,math,exec]'`. Heavy execution environments never enter the venv: code benchmarks run in
docker sandboxes (`python:3.11-slim`, the official bigcodebench image,
Dependency tiers (deliberate — no monolithic image): text-compare recipes `sweb.eval.*` per-instance images via `sandbox prefetch`).
run on core; symbolic math needs `[math]`; DROP needs `[exec]`; code-execution
benchmarks (humaneval/bigcodebench/LCB/bfcl) additionally require the sandbox
layer (roadmap) since they run model-generated code in isolation.
## Quick start ## Quick start

2
evalharness/data/loader.py
View File

@ -386,7 +386,7 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]:
except ImportError: except ImportError:
raise ImportError( raise ImportError(
f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' f'dataset {spec.name!r} lives on a hub ({spec.source!r}); '
'install it first: pip install "evalharness[all]" (or [hub] for datasets only)' 'install it first: pip install evalharness (light deps are default)'
) )
kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS}
subset = None if spec.subset == 'default' else spec.subset subset = None if spec.subset == 'default' else spec.subset

7
evalharness/eval/scorer.py
View File

@ -129,7 +129,7 @@ def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext):
details['acc'] = {'path': 'sympy'} details['acc'] = {'path': 'sympy'}
return {'acc': 1.0}, details return {'acc': 1.0}, details
except ImportError: except ImportError:
details['acc']['sympy'] = 'not installed; pip install "evalharness[all]" enables official symbolic grading' details['acc']['sympy'] = 'not installed; reinstall evalharness for official symbolic grading'
return {'acc': 0.0}, details return {'acc': 0.0}, details
@ -194,7 +194,7 @@ def _drop_metrics(predicted: List[str], gold: List[str]):
import numpy as np import numpy as np
from scipy.optimize import linear_sum_assignment from scipy.optimize import linear_sum_assignment
except ImportError as e: except ImportError as e:
raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[all]'") from e raise LayerNotReady("official DROP scoring needs numpy+scipy (default deps); reinstall evalharness") from e
n, m = len(gold_bags), len(pred_bags) n, m = len(gold_bags), len(pred_bags)
if m == 0: if m == 0:
@ -358,8 +358,7 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
raise LayerNotReady(f'backend {backend!r} unavailable ' raise LayerNotReady(f'backend {backend!r} unavailable '
'(missing dependency?)') '(missing dependency?)')
# auto fell back: say so in details so users know native was used # auto fell back: say so in details so users know native was used
native_details_hint = ('native comparison used; install "evalharness[bfcl]" ' native_details_hint = ('native comparison used; install "evalharness[bfcl]" for the official ast_checker')
'for the official ast_checker')
gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls') gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls')
if gt_calls is None: if gt_calls is None:
# irrelevance categories: correct behavior is calling NOTHING # irrelevance categories: correct behavior is calling NOTHING

19
pyproject.toml
View File

@ -7,16 +7,19 @@ name = "evalharness"
version = "0.1.0" version = "0.1.0"
description = "A plugin-based LLM/agent evaluation harness (data layer first)" description = "A plugin-based LLM/agent evaluation harness (data layer first)"
requires-python = ">=3.10" requires-python = ">=3.10"
dependencies = ["pydantic>=2"] dependencies = [
"pydantic>=2",
"datasets", # HuggingFace-hosted datasets (light, conflict-free)
"pyarrow", # parquet sources (ModelScope/HF raw mirrors)
"sympy", # official PRM800K symbolic math grading
"pylatexenc",
"numpy", # official DROP aligner
"scipy",
]
[project.optional-dependencies] [project.optional-dependencies]
hub = ["datasets"] # HuggingFace-hosted datasets bfcl = ["bfcl-eval", "soundfile"] # heavy: official bfcl ast_checker (pulls 5 cloud SDKs
parquet = ["pyarrow"] # parquet sources (ModelScope/HF raw mirrors) # + qwen-agent); native scorer is the default
math = ["sympy", "pylatexenc"] # symbolic math grading (aime/math family), official PRM800K logic
exec = ["numpy", "scipy"] # DROP official aligner (linear_sum_assignment)
bfcl = ["bfcl-eval", "soundfile"] # OFFICIAL bfcl ast_checker (heavy: pulls 5 cloud SDKs +
# qwen-agent; native scorer is the dependency-free default)
all = ["datasets", "pyarrow", "sympy", "pylatexenc", "numpy", "scipy"] # everything conflict-free
[project.scripts] [project.scripts]
evalharness = "evalharness.cli:main" evalharness = "evalharness.cli:main"