370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
36 lines
828 B
Python
36 lines
828 B
Python
"""Narration themes: the icon/word/color mapping for progress lines.
|
|
|
|
A theme is a narrate(msg) -> rich-markup-string function plus its name;
|
|
the CLI picks one with --theme (default 'default'). Drop a module in
|
|
evalharness/themes/ and it registers itself.
|
|
"""
|
|
from typing import Callable
|
|
|
|
from ..eval.registry import EvalRegistry
|
|
|
|
THEME_REGISTRY = EvalRegistry('narration theme')
|
|
|
|
|
|
def register_theme(name: str):
|
|
def decorator(fn: Callable[[str], str]):
|
|
THEME_REGISTRY.register(name, fn)
|
|
return fn
|
|
|
|
return decorator
|
|
|
|
|
|
def get_theme(name: str = 'default'):
|
|
return THEME_REGISTRY.get(name)
|
|
|
|
|
|
def _discover():
|
|
import importlib
|
|
import pkgutil
|
|
|
|
for m in pkgutil.iter_modules(__path__):
|
|
if m.name != '__init__':
|
|
importlib.import_module(f'{__name__}.{m.name}')
|
|
|
|
|
|
_discover()
|