370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
25 lines
582 B
Python
25 lines
582 B
Python
"""Endpoint prober plugins: how the runner verifies a model endpoint is
|
|
usable before burning samples. Default 'ping' sends one 1-token request."""
|
|
from ..eval.registry import EvalRegistry
|
|
|
|
PROBER_REGISTRY = EvalRegistry('prober')
|
|
|
|
|
|
def register_prober(name: str):
|
|
def decorator(fn):
|
|
PROBER_REGISTRY.register(name, fn)
|
|
return fn
|
|
|
|
return decorator
|
|
|
|
|
|
def get_prober(name: str = 'ping'):
|
|
return PROBER_REGISTRY.get(name)
|
|
|
|
|
|
@register_prober('ping')
|
|
async def ping(adapter):
|
|
from .runner import _default_ping_probe
|
|
|
|
await _default_ping_probe(adapter)
|