Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unify on-demand install hints to extras names ([all]/[bfcl]); native bfcl fallback carries upgrade hint

Browse Source
This commit is contained in:
sora 2026-08-25 02:49:09 +00:00
parent 32b7276142
commit 99c9df5412
2 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
evalharness/data/loader.py
View File

@ -106,7 +106,7 @@ def _read_file(path: str) -> List[Dict[str, Any]]:
import pyarrow.parquet as pq import pyarrow.parquet as pq
except ImportError: except ImportError:
raise ImportError( raise ImportError(
f'{path} is parquet; install the reader first: pip install pyarrow' f'{path} is parquet; install the reader first: pip install "evalharness[all]"'
) )
return pq.read_table(path).to_pylist() return pq.read_table(path).to_pylist()
with open(path, encoding='utf-8') as f: with open(path, encoding='utf-8') as f:
@ -386,7 +386,7 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]:
except ImportError: except ImportError:
raise ImportError( raise ImportError(
f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' f'dataset {spec.name!r} lives on a hub ({spec.source!r}); '
"install the optional dependency first: pip install 'evalharness[hub]'" 'install it first: pip install "evalharness[all]" (or [hub] for datasets only)'
) )
kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS}
subset = None if spec.subset == 'default' else spec.subset subset = None if spec.subset == 'default' else spec.subset

11
evalharness/eval/scorer.py
View File

@ -129,7 +129,7 @@ def math_equal(pred: str, target, sample: Sample, ctx: ScoreContext):
details['acc'] = {'path': 'sympy'} details['acc'] = {'path': 'sympy'}
return {'acc': 1.0}, details return {'acc': 1.0}, details
except ImportError: except ImportError:
details['acc']['sympy'] = 'not installed (pip install sympy pylatexenc)' details['acc']['sympy'] = 'not installed; pip install "evalharness[all]" enables official symbolic grading'
return {'acc': 0.0}, details return {'acc': 0.0}, details
@ -194,7 +194,7 @@ def _drop_metrics(predicted: List[str], gold: List[str]):
import numpy as np import numpy as np
from scipy.optimize import linear_sum_assignment from scipy.optimize import linear_sum_assignment
except ImportError as e: except ImportError as e:
raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[exec]'") from e raise LayerNotReady("official DROP scoring needs numpy+scipy: pip install 'evalharness[all]'") from e
n, m = len(gold_bags), len(pred_bags) n, m = len(gold_bags), len(pred_bags)
if m == 0: if m == 0:
@ -357,11 +357,15 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
if backend != 'auto': if backend != 'auto':
raise LayerNotReady(f'backend {backend!r} unavailable ' raise LayerNotReady(f'backend {backend!r} unavailable '
'(missing dependency?)') '(missing dependency?)')
# auto fell back: say so in details so users know native was used
native_details_hint = ('native comparison used; install "evalharness[bfcl]" '
'for the official ast_checker')
gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls') gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls')
if gt_calls is None: if gt_calls is None:
# irrelevance categories: correct behavior is calling NOTHING # irrelevance categories: correct behavior is calling NOTHING
hit = int(len(calls) == 0) hit = int(len(calls) == 0)
return {'acc': float(hit)}, {'acc': {'mode': 'no_calls', 'n_calls': len(calls)}} return {'acc': float(hit)}, {'acc': {'mode': 'no_calls', 'n_calls': len(calls),
'hint': native_details_hint}}
def norm(call: Dict[str, Any]) -> str: def norm(call: Dict[str, Any]) -> str:
return json.dumps({'name': call.get('name'), return json.dumps({'name': call.get('name'),
@ -374,6 +378,7 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
return {'acc': float(hit)}, {'acc': { return {'acc': float(hit)}, {'acc': {
'mode': 'call_sequence', 'expected': want[:5], 'got': got[:5], 'mode': 'call_sequence', 'expected': want[:5], 'got': got[:5],
'n_expected': len(want), 'n_got': len(got), 'n_expected': len(want), 'n_got': len(got),
'hint': native_details_hint,
}} }}