From 89e721414fb7145bcaedde70bcb2ef1e166863d7 Mon Sep 17 00:00:00 2001 From: sora Date: Tue, 25 Aug 2026 02:51:55 +0000 Subject: [PATCH] Simplify install: light deps (datasets/pyarrow/sympy/pylatexenc/numpy/scipy) are DEFAULT; single [bfcl] extra for the heavy official checker --- README.md | 18 +++++++----------- evalharness/data/loader.py | 2 +- evalharness/eval/scorer.py | 7 +++---- pyproject.toml | 19 +++++++++++-------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index cac53ae..25cd780 100644 --- a/README.md +++ b/README.md @@ -30,19 +30,15 @@ land next. ## Install ```bash -pip install . # core only (pydantic) -pip install '.[hub]' # + HuggingFace datasets -pip install '.[math]' # + sympy/pylatexenc (aime/math official grader) -pip install '.[exec]' # + numpy/scipy (official DROP aligner) -pip install '.[hub,parquet,math,exec]' # everything +pip install . # core + all light deps (datasets/pyarrow/sympy/numpy/scipy): + # every benchmark scores officially, nothing conflicts +pip install '.[bfcl]' # + heavy official bfcl ast_checker (5 cloud SDKs + qwen-agent; + # only for official-parity BFCL scoring -- native default works) ``` -For development: `pip install -e '.[hub,parquet,math,exec]'`. - -Dependency tiers (deliberate — no monolithic image): text-compare recipes -run on core; symbolic math needs `[math]`; DROP needs `[exec]`; code-execution -benchmarks (humaneval/bigcodebench/LCB/bfcl) additionally require the sandbox -layer (roadmap) since they run model-generated code in isolation. +Heavy execution environments never enter the venv: code benchmarks run in +docker sandboxes (`python:3.11-slim`, the official bigcodebench image, +`sweb.eval.*` per-instance images via `sandbox prefetch`). ## Quick start diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py index 420d00c..2068a20 100644 --- a/evalharness/data/loader.py +++ b/evalharness/data/loader.py @@ -386,7 +386,7 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]: except ImportError: raise ImportError( f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' - 'install it first: pip install "evalharness[all]" (or [hub] for datasets only)' + 'install it first: pip install evalharness (light deps are default)' ) kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} subset = None if spec.subset == 'default' else spec.subset diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index cb66f6f..a1e5700 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -129,7 +129,7 @@ def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext): details['acc'] = {'path': 'sympy'} return {'acc': 1.0}, details except ImportError: - details['acc']['sympy'] = 'not installed; pip install "evalharness[all]" enables official symbolic grading' + details['acc']['sympy'] = 'not installed; reinstall evalharness for official symbolic grading' return {'acc': 0.0}, details @@ -194,7 +194,7 @@ def _drop_metrics(predicted: List[str], gold: List[str]): import numpy as np from scipy.optimize import linear_sum_assignment except ImportError as e: - raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[all]'") from e + raise LayerNotReady("official DROP scoring needs numpy+scipy (default deps); reinstall evalharness") from e n, m = len(gold_bags), len(pred_bags) if m == 0: @@ -358,8 +358,7 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext): raise LayerNotReady(f'backend {backend!r} unavailable ' '(missing dependency?)') # auto fell back: say so in details so users know native was used - native_details_hint = ('native comparison used; install "evalharness[bfcl]" ' - 'for the official ast_checker') + native_details_hint = ('native comparison used; install "evalharness[bfcl]" for the official ast_checker') gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls') if gt_calls is None: # irrelevance categories: correct behavior is calling NOTHING diff --git a/pyproject.toml b/pyproject.toml index 8f51fe7..4bec278 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,16 +7,19 @@ name = "evalharness" version = "0.1.0" description = "A plugin-based LLM/agent evaluation harness (data layer first)" requires-python = ">=3.10" -dependencies = ["pydantic>=2"] +dependencies = [ + "pydantic>=2", + "datasets", # HuggingFace-hosted datasets (light, conflict-free) + "pyarrow", # parquet sources (ModelScope/HF raw mirrors) + "sympy", # official PRM800K symbolic math grading + "pylatexenc", + "numpy", # official DROP aligner + "scipy", +] [project.optional-dependencies] -hub = ["datasets"] # HuggingFace-hosted datasets -parquet = ["pyarrow"] # parquet sources (ModelScope/HF raw mirrors) -math = ["sympy", "pylatexenc"] # symbolic math grading (aime/math family), official PRM800K logic -exec = ["numpy", "scipy"] # DROP official aligner (linear_sum_assignment) -bfcl = ["bfcl-eval", "soundfile"] # OFFICIAL bfcl ast_checker (heavy: pulls 5 cloud SDKs + - # qwen-agent; native scorer is the dependency-free default) -all = ["datasets", "pyarrow", "sympy", "pylatexenc", "numpy", "scipy"] # everything conflict-free +bfcl = ["bfcl-eval", "soundfile"] # heavy: official bfcl ast_checker (pulls 5 cloud SDKs + # + qwen-agent); native scorer is the default [project.scripts] evalharness = "evalharness.cli:main"