370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
48 lines
1.8 KiB
Python
48 lines
1.8 KiB
Python
"""evalharness.eval -- the evaluation layer.
|
|
|
|
Pipeline: extract -> score -> aggregate, all plugin-driven.
|
|
|
|
from evalharness.eval import evaluate, get_eval
|
|
from evalharness import get_dataset
|
|
|
|
ds = get_dataset('gsm8k')
|
|
report = evaluate(ds, predictions) # recipe auto-resolved by dataset name
|
|
report.save('gsm8k.report.json')
|
|
|
|
Four scoring paradigms: text-compare (implemented), llm-judge (wired via
|
|
runner judge= once ModelAdapter exists), execution & env-reward (slots raise
|
|
LayerNotReady until sandbox/agent layers land).
|
|
"""
|
|
|
|
from .aggregator import AGGREGATOR_REGISTRY, get_aggregator, register_aggregator
|
|
from .extractor import EXTRACTOR_REGISTRY, get_extractor, make_extractor, register_extractor
|
|
from .recipe import EVAL_REGISTRY, EvalRecipe, JudgeConfig, get_eval, list_evals, register_eval
|
|
from .record import EvalReport, SampleResult
|
|
from .registry import EvalRegistry
|
|
from .runner import evaluate
|
|
from .scorer import SCORER_REGISTRY, LayerNotReady, ScoreContext, get_scorer, register_scorer
|
|
|
|
__all__ = [
|
|
'evaluate', 'EvalRecipe', 'JudgeConfig', 'get_eval', 'list_evals', 'register_eval',
|
|
'EvalReport', 'SampleResult', 'LayerNotReady', 'ScoreContext',
|
|
'EXTRACTOR_REGISTRY', 'SCORER_REGISTRY', 'AGGREGATOR_REGISTRY', 'EvalRegistry',
|
|
'register_extractor', 'get_extractor', 'make_extractor',
|
|
'register_scorer', 'get_scorer', 'register_aggregator', 'get_aggregator',
|
|
]
|
|
|
|
|
|
def _discover_builtin_recipes() -> None:
|
|
"""Import every recipe module under ./recipes (import = register)."""
|
|
import importlib
|
|
import pkgutil
|
|
from pathlib import Path
|
|
|
|
pkg_dir = Path(__file__).parent / 'recipes'
|
|
if not pkg_dir.exists():
|
|
return
|
|
for info in pkgutil.iter_modules([str(pkg_dir)]):
|
|
importlib.import_module(f'{__name__}.recipes.{info.name}')
|
|
|
|
|
|
_discover_builtin_recipes()
|