Simplify install: light deps (datasets/pyarrow/sympy/pylatexenc/numpy/scipy) are DEFAULT; single [bfcl] extra for the heavy official checker

README.md

@@ -30,19 +30,15 @@ land next.
 ## Install
 
 ```bash
-pip install .                    # core only (pydantic)
-pip install '.[hub]'             # + HuggingFace datasets
-pip install '.[math]'            # + sympy/pylatexenc (aime/math official grader)
-pip install '.[exec]'            # + numpy/scipy (official DROP aligner)
-pip install '.[hub,parquet,math,exec]'  # everything
+pip install .                  # core + all light deps (datasets/pyarrow/sympy/numpy/scipy):
+                               # every benchmark scores officially, nothing conflicts
+pip install '.[bfcl]'          # + heavy official bfcl ast_checker (5 cloud SDKs + qwen-agent;
+                               # only for official-parity BFCL scoring -- native default works)
 ```
 
-For development: `pip install -e '.[hub,parquet,math,exec]'`.
-
-Dependency tiers (deliberate — no monolithic image): text-compare recipes
-run on core; symbolic math needs `[math]`; DROP needs `[exec]`; code-execution
-benchmarks (humaneval/bigcodebench/LCB/bfcl) additionally require the sandbox
-layer (roadmap) since they run model-generated code in isolation.
+Heavy execution environments never enter the venv: code benchmarks run in
+docker sandboxes (`python:3.11-slim`, the official bigcodebench image,
+`sweb.eval.*` per-instance images via `sandbox prefetch`).
 
 ## Quick start
 

evalharness/data/loader.py

@@ -386,7 +386,7 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]:
     except ImportError:
         raise ImportError(
             f'dataset {spec.name!r} lives on a hub ({spec.source!r}); '
-            'install it first: pip install "evalharness[all]" (or [hub] for datasets only)'
+            'install it first: pip install evalharness (light deps are default)'
         )
     kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS}
     subset = None if spec.subset == 'default' else spec.subset

evalharness/eval/scorer.py

@@ -129,7 +129,7 @@ def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext):
                 details['acc'] = {'path': 'sympy'}
                 return {'acc': 1.0}, details
         except ImportError:
-            details['acc']['sympy'] = 'not installed; pip install "evalharness[all]" enables official symbolic grading'
+            details['acc']['sympy'] = 'not installed; reinstall evalharness for official symbolic grading'
     return {'acc': 0.0}, details
 
 
@@ -194,7 +194,7 @@ def _drop_metrics(predicted: List[str], gold: List[str]):
         import numpy as np
         from scipy.optimize import linear_sum_assignment
     except ImportError as e:
-        raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[all]'") from e
+        raise LayerNotReady("official DROP scoring needs numpy+scipy (default deps); reinstall evalharness") from e
 
     n, m = len(gold_bags), len(pred_bags)
     if m == 0:
@@ -358,8 +358,7 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
             raise LayerNotReady(f'backend {backend!r} unavailable '
                                 '(missing dependency?)')
         # auto fell back: say so in details so users know native was used
-        native_details_hint = ('native comparison used; install "evalharness[bfcl]" '
-                               'for the official ast_checker')
+        native_details_hint = ('native comparison used; install "evalharness[bfcl]" for the official ast_checker')
     gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls')
     if gt_calls is None:
         # irrelevance categories: correct behavior is calling NOTHING

pyproject.toml

@@ -7,16 +7,19 @@ name = "evalharness"
 version = "0.1.0"
 description = "A plugin-based LLM/agent evaluation harness (data layer first)"
 requires-python = ">=3.10"
-dependencies = ["pydantic>=2"]
+dependencies = [
+    "pydantic>=2",
+    "datasets",       # HuggingFace-hosted datasets (light, conflict-free)
+    "pyarrow",        # parquet sources (ModelScope/HF raw mirrors)
+    "sympy",          # official PRM800K symbolic math grading
+    "pylatexenc",
+    "numpy",          # official DROP aligner
+    "scipy",
+]
 
 [project.optional-dependencies]
-hub = ["datasets"]            # HuggingFace-hosted datasets
-parquet = ["pyarrow"]         # parquet sources (ModelScope/HF raw mirrors)
-math = ["sympy", "pylatexenc"]  # symbolic math grading (aime/math family), official PRM800K logic
-exec = ["numpy", "scipy"]     # DROP official aligner (linear_sum_assignment)
-bfcl = ["bfcl-eval", "soundfile"]  # OFFICIAL bfcl ast_checker (heavy: pulls 5 cloud SDKs +
-                                   # qwen-agent; native scorer is the dependency-free default)
-all = ["datasets", "pyarrow", "sympy", "pylatexenc", "numpy", "scipy"]  # everything conflict-free
+bfcl = ["bfcl-eval", "soundfile"]  # heavy: official bfcl ast_checker (pulls 5 cloud SDKs
+                                   # + qwen-agent); native scorer is the default
 
 [project.scripts]
 evalharness = "evalharness.cli:main"