370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
"""Shared tiny registry for eval-layer plugins (mirrors data registry semantics)."""
|
|
|
|
import difflib
|
|
from typing import Callable, Dict, List
|
|
|
|
|
|
class EvalRegistry:
|
|
"""Dict registry: decorator registration, duplicate guard, suggestions."""
|
|
|
|
def __init__(self, kind: str):
|
|
self.kind = kind
|
|
self._items: Dict[str, Callable] = {}
|
|
|
|
def register(self, name: str, fn: Callable) -> Callable:
|
|
if name in self._items:
|
|
raise ValueError(f'{self.kind} {name!r} is already registered')
|
|
self._items[name] = fn
|
|
return fn
|
|
|
|
def get(self, name: str) -> Callable:
|
|
if name not in self._items:
|
|
near = difflib.get_close_matches(name, self._items, n=3)
|
|
hint = f" Did you mean: {', '.join(near)}?" if near else ''
|
|
raise KeyError(f'unknown {self.kind} {name!r}.{hint}')
|
|
return self._items[name]
|
|
|
|
def names(self) -> List[str]:
|
|
return sorted(self._items)
|
|
|
|
def __contains__(self, name: str) -> bool:
|
|
return name in self._items
|
|
|
|
def __len__(self) -> int:
|
|
return len(self._items)
|