"""Run-lifecycle hooks: plugins that observe/extend a run without touching its logic. Register with @register_hook('on_benchmark_failed') etc.; the CLI fires them at the matching points. Multiple hooks per event all run. from evalharness.hooks import register_hook @register_hook('on_benchmark_failed') def notify(name, error, **ctx): requests.post(webhook, json={'bench': name, 'error': str(error)}) """ from typing import Callable, Dict, List _HOOKS: Dict[str, List[Callable]] = {} def register_hook(event: str): def decorator(fn): _HOOKS.setdefault(event, []).append(fn) return fn return decorator def fire(event: str, **ctx) -> None: """Invoke every hook registered for the event; hook errors are printed and swallowed -- observability must never break the run.""" for fn in _HOOKS.get(event, []): try: fn(**ctx) except Exception as e: print(f'hook {fn.__name__!r} failed: {type(e).__name__}: {e}', flush=True)