370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
62 lines
1.1 KiB
Python
62 lines
1.1 KiB
Python
"""Bar-less progress reporter: narration lines only (terminals without
|
|
rich, CI logs, JSON-adjacent consumers)."""
|
|
from ..eval.registry import EvalRegistry
|
|
|
|
PROGRESS_REGISTRY = EvalRegistry('progress reporter')
|
|
|
|
|
|
def register_progress(name: str):
|
|
def decorator(cls):
|
|
PROGRESS_REGISTRY.register(name, cls)
|
|
return cls
|
|
|
|
return decorator
|
|
|
|
|
|
@register_progress('plain')
|
|
class PlainProgress:
|
|
name = 'plain'
|
|
|
|
def __init__(self, console=None):
|
|
self.console = console
|
|
|
|
def log(self, message):
|
|
print(message, flush=True)
|
|
|
|
# the rest are no-ops: no bars, no per-sample accounting
|
|
def set_overall(self, *a, **k):
|
|
pass
|
|
|
|
def advance_overall(self):
|
|
pass
|
|
|
|
def start(self, *a, **k):
|
|
pass
|
|
|
|
def reset_samples(self, *a, **k):
|
|
pass
|
|
|
|
def set_bench_tag(self, *a, **k):
|
|
pass
|
|
|
|
def set_phase(self, *a, **k):
|
|
pass
|
|
|
|
def begin_sample(self, *a, **k):
|
|
pass
|
|
|
|
def rollback(self):
|
|
pass
|
|
|
|
def advance(self, *a, **k):
|
|
pass
|
|
|
|
def pause(self):
|
|
pass
|
|
|
|
def resume(self):
|
|
pass
|
|
|
|
def close(self):
|
|
pass
|