370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
32 lines
1019 B
Python
32 lines
1019 B
Python
"""Run-lifecycle hooks: plugins that observe/extend a run without touching
|
|
its logic. Register with @register_hook('on_benchmark_failed') etc.; the CLI
|
|
fires them at the matching points. Multiple hooks per event all run.
|
|
|
|
from evalharness.hooks import register_hook
|
|
|
|
@register_hook('on_benchmark_failed')
|
|
def notify(name, error, **ctx):
|
|
requests.post(webhook, json={'bench': name, 'error': str(error)})
|
|
"""
|
|
from typing import Callable, Dict, List
|
|
|
|
_HOOKS: Dict[str, List[Callable]] = {}
|
|
|
|
|
|
def register_hook(event: str):
|
|
def decorator(fn):
|
|
_HOOKS.setdefault(event, []).append(fn)
|
|
return fn
|
|
|
|
return decorator
|
|
|
|
|
|
def fire(event: str, **ctx) -> None:
|
|
"""Invoke every hook registered for the event; hook errors are printed
|
|
and swallowed -- observability must never break the run."""
|
|
for fn in _HOOKS.get(event, []):
|
|
try:
|
|
fn(**ctx)
|
|
except Exception as e:
|
|
print(f'hook {fn.__name__!r} failed: {type(e).__name__}: {e}', flush=True)
|