370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
86 lines
2.9 KiB
Python
86 lines
2.9 KiB
Python
"""EvalRecipe: what a benchmark's evaluation IS, declared not coded.
|
|
|
|
Mirrors the data-layer plugin shape: a recipe binds
|
|
extract -- extractor spec (primitive name / cascade / custom fn)
|
|
scorers -- {metric: scorer spec}; spec = name | fn | {'name': ..., **params}
|
|
aggregators -- {metric: aggregator name | (name, params)} (default 'mean')
|
|
judge -- optional LLM-judge config (model tag; wiring comes later)
|
|
|
|
Registry: @register_eval('gsm8k') -> get_eval('gsm8k') -> EvalRecipe
|
|
"""
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
|
|
from .aggregator import get_aggregator
|
|
from .extractor import ExtractorSpec, make_extractor
|
|
from .scorer import ScorerSpec, make_scorer
|
|
from .registry import EvalRegistry
|
|
|
|
EVAL_REGISTRY = EvalRegistry('eval recipe')
|
|
|
|
|
|
def register_eval(name: str):
|
|
def decorator(factory: Callable[[], 'EvalRecipe']):
|
|
EVAL_REGISTRY.register(name, factory)
|
|
return factory
|
|
|
|
return decorator
|
|
|
|
|
|
def get_eval(name: str) -> 'EvalRecipe':
|
|
return EVAL_REGISTRY.get(name)()
|
|
|
|
|
|
def list_evals() -> List[str]:
|
|
return EVAL_REGISTRY.names()
|
|
|
|
|
|
@dataclass
|
|
class JudgeConfig:
|
|
"""LLM-judge wiring; the actual callable is injected by the runner."""
|
|
|
|
model: str = '' # model tag / url, resolved by ModelAdapter later
|
|
temperature: float = 0.0
|
|
max_retries: int = 2
|
|
|
|
|
|
@dataclass
|
|
class EvalRecipe:
|
|
name: str = ''
|
|
extract: ExtractorSpec = None
|
|
scorers: Dict[str, ScorerSpec] = field(default_factory=dict)
|
|
# metric -> aggregator name | (name, params); missing -> 'mean'
|
|
aggregators: Dict[str, Union[str, Tuple[str, Dict[str, Any]]]] = field(default_factory=dict)
|
|
judge: Optional[JudgeConfig] = None
|
|
description: str = ''
|
|
exec_workers: int = 1 # parallel judging threads (docker/subprocess
|
|
# execution benches: 8-12; llm_judge stays 1 unless
|
|
# the judge endpoint can take it)
|
|
|
|
def resolve_extract(self):
|
|
return make_extractor(self.extract)
|
|
|
|
def resolve_scorers(self) -> Dict[str, Callable]:
|
|
if not self.scorers:
|
|
raise ValueError(f'recipe {self.name!r} has no scorers')
|
|
return {m: make_scorer(m, s) for m, s in self.scorers.items()}
|
|
|
|
def resolve_aggregators(self) -> Dict[str, Callable]:
|
|
out = {}
|
|
for metric, spec in self.aggregators.items():
|
|
if isinstance(spec, tuple):
|
|
name, params = spec
|
|
base = get_aggregator(name)
|
|
|
|
def with_params(results, m, _base=base, _params=params):
|
|
return _base(results, m, **_params) if _params else _base(results, m)
|
|
|
|
out[metric] = with_params
|
|
else:
|
|
out[metric] = get_aggregator(spec or 'mean')
|
|
return out
|
|
|
|
def primary_metric(self) -> str:
|
|
return next(iter(self.scorers), 'acc')
|