diff --git a/evalharness/cli.py b/evalharness/cli.py index b6e7b44..72222d9 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -728,7 +728,34 @@ def _cmd_eval_run(args) -> int: _rep_secs = 0.0 _rep_tin = _rep_tout = 0 - if model_spec: # generate + score in one go + report = None + # score cache: the checkpoint stores PREDICTIONS, not scores -- + # but when every prediction is already checkpointed AND a saved + # report matches (same model, same sample count), re-scoring + # reproduces the same numbers, so reuse the report and skip the + # whole docker/exec scoring pass. --rescore forces evaluation + # (recipe/judge changed, or just paranoia). + if model_spec and out_dir and _repeats == 1 \ + and not getattr(args, 'rescore', False): + from pathlib import Path as _P2 + + _rp = _P2(out_dir) / name / 'report.jsonl' + if _rp.exists(): + try: + from evalharness.eval.record import EvalReport as _ER + + _old = _ER.load(str(_rp)) + _m_old = (_old.model or '').split('?')[-1] + _m_new = model_spec.split('?')[-1] + if _m_old == _m_new \ + and _old.num_samples == (args.limit or sample_count): + report = _old + _emit('Reusing saved report -- predictions all ' + 'checkpointed (--rescore to re-evaluate)') + except Exception: + pass # unreadable/stale report: score normally + + if report is None and model_spec: # generate + score in one go from evalharness.model import run_eval progress_reporter = None @@ -861,7 +888,7 @@ def _cmd_eval_run(args) -> int: if _primary: report.metrics[f'{_primary}_last_run'] = report.metrics[_primary] report.metrics[_primary] = _mean - else: + elif report is None: from evalharness.eval import evaluate if not args.predictions: @@ -1166,6 +1193,9 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--resume', nargs='?', const=True, default=False, help='resume from per-sample checkpoint (default path auto-derived; ' 'pass a path to override)') + p.add_argument('--rescore', action='store_true', + help='force re-scoring even when a matching saved report ' + 'could be reused (change of recipe/judge, or paranoia)') p.add_argument('--limit-per-task', type=int, help='first N samples PER subset/category (evalscope --limit semantics); ' 'composable with --limit (intersection)') diff --git a/evalharness/eval/record.py b/evalharness/eval/record.py index 62370cd..72edf98 100644 --- a/evalharness/eval/record.py +++ b/evalharness/eval/record.py @@ -56,7 +56,10 @@ class EvalReport(BaseModel): num_samples: int = 0 num_failed_extractions: int = 0 metrics: Dict[str, float] = Field(default_factory=dict) # {'acc': 0.62} - metric_groups: Dict[str, Dict[str, float]] = Field(default_factory=dict) + # values may be None (perf stats the adapter couldn't measure), lists + # (repeats.scores) or nested dicts -- a strict float type rejected the + # file on LOAD and silently defeated report reuse + metric_groups: Dict[str, Dict[str, Any]] = Field(default_factory=dict) # {'by_category': {'algebra': 0.7, ...}, 'pass_at_k': {'pass@1': .., 'pass@8': ..}, # 'by_length_bin': {'8k': .., '32k': ..}}