diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py index 14fd930..420d00c 100644 --- a/evalharness/data/loader.py +++ b/evalharness/data/loader.py @@ -106,7 +106,7 @@ def _read_file(path: str) -> List[Dict[str, Any]]: import pyarrow.parquet as pq except ImportError: raise ImportError( - f'{path} is parquet; install the reader first: pip install pyarrow' + f'{path} is parquet; install the reader first: pip install "evalharness[all]"' ) return pq.read_table(path).to_pylist() with open(path, encoding='utf-8') as f: @@ -386,7 +386,7 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]: except ImportError: raise ImportError( f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' - "install the optional dependency first: pip install 'evalharness[hub]'" + 'install it first: pip install "evalharness[all]" (or [hub] for datasets only)' ) kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} subset = None if spec.subset == 'default' else spec.subset diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index 4af5dfe..cb66f6f 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -129,7 +129,7 @@ def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext): details['acc'] = {'path': 'sympy'} return {'acc': 1.0}, details except ImportError: - details['acc']['sympy'] = 'not installed (pip install sympy pylatexenc)' + details['acc']['sympy'] = 'not installed; pip install "evalharness[all]" enables official symbolic grading' return {'acc': 0.0}, details @@ -194,7 +194,7 @@ def _drop_metrics(predicted: List[str], gold: List[str]): import numpy as np from scipy.optimize import linear_sum_assignment except ImportError as e: - raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[exec]'") from e + raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[all]'") from e n, m = len(gold_bags), len(pred_bags) if m == 0: @@ -357,11 +357,15 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext): if backend != 'auto': raise LayerNotReady(f'backend {backend!r} unavailable ' '(missing dependency?)') + # auto fell back: say so in details so users know native was used + native_details_hint = ('native comparison used; install "evalharness[bfcl]" ' + 'for the official ast_checker') gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls') if gt_calls is None: # irrelevance categories: correct behavior is calling NOTHING hit = int(len(calls) == 0) - return {'acc': float(hit)}, {'acc': {'mode': 'no_calls', 'n_calls': len(calls)}} + return {'acc': float(hit)}, {'acc': {'mode': 'no_calls', 'n_calls': len(calls), + 'hint': native_details_hint}} def norm(call: Dict[str, Any]) -> str: return json.dumps({'name': call.get('name'), @@ -374,6 +378,7 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext): return {'acc': float(hit)}, {'acc': { 'mode': 'call_sequence', 'expected': want[:5], 'got': got[:5], 'n_expected': len(want), 'n_got': len(got), + 'hint': native_details_hint, }}