From f272146b54362fe8bf1b6dcd55fd81a1527778b4 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Mon, 14 Sep 2026 06:12:30 +0000
Subject: [PATCH] Report reuse: skip re-scoring when all predictions are
 checkpointed

The checkpoint stores predictions, not scores -- reruns re-executed the
whole docker scoring pass even when nothing could change. When the
saved report matches (same model, same sample count) and every
prediction is checkpointed, reuse it; --rescore forces evaluation
(recipe/judge changed).

Also fixes EvalReport.load: metric_groups declared Dict[str, float]
rejected the None perf values (and the repeats scores list) the file
format actually contains -- every load raised ValidationError, which
is what silently killed reuse in testing.

Repeats>1 benches never reuse: their whole point is fresh sampling.

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/cli.py         | 34 ++++++++++++++++++++++++++++++++--
 evalharness/eval/record.py |  5 ++++-
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/evalharness/cli.py b/evalharness/cli.py
index b6e7b44..72222d9 100644
--- a/evalharness/cli.py
+++ b/evalharness/cli.py
@@ -728,7 +728,34 @@ def _cmd_eval_run(args) -> int:
             _rep_secs = 0.0
             _rep_tin = _rep_tout = 0
 
-            if model_spec:  # generate + score in one go
+            report = None
+            # score cache: the checkpoint stores PREDICTIONS, not scores --
+            # but when every prediction is already checkpointed AND a saved
+            # report matches (same model, same sample count), re-scoring
+            # reproduces the same numbers, so reuse the report and skip the
+            # whole docker/exec scoring pass. --rescore forces evaluation
+            # (recipe/judge changed, or just paranoia).
+            if model_spec and out_dir and _repeats == 1 \
+                    and not getattr(args, 'rescore', False):
+                from pathlib import Path as _P2
+
+                _rp = _P2(out_dir) / name / 'report.jsonl'
+                if _rp.exists():
+                    try:
+                        from evalharness.eval.record import EvalReport as _ER
+
+                        _old = _ER.load(str(_rp))
+                        _m_old = (_old.model or '').split('?')[-1]
+                        _m_new = model_spec.split('?')[-1]
+                        if _m_old == _m_new \
+                                and _old.num_samples == (args.limit or sample_count):
+                            report = _old
+                            _emit('Reusing saved report -- predictions all '
+                                  'checkpointed (--rescore to re-evaluate)')
+                    except Exception:
+                        pass  # unreadable/stale report: score normally
+
+            if report is None and model_spec:  # generate + score in one go
                 from evalharness.model import run_eval
 
                 progress_reporter = None
@@ -861,7 +888,7 @@ def _cmd_eval_run(args) -> int:
                     if _primary:
                         report.metrics[f'{_primary}_last_run'] = report.metrics[_primary]
                         report.metrics[_primary] = _mean
-            else:
+            elif report is None:
                 from evalharness.eval import evaluate
 
                 if not args.predictions:
@@ -1166,6 +1193,9 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument('--resume', nargs='?', const=True, default=False,
                    help='resume from per-sample checkpoint (default path auto-derived; '
                         'pass a path to override)')
+    p.add_argument('--rescore', action='store_true',
+                   help='force re-scoring even when a matching saved report '
+                        'could be reused (change of recipe/judge, or paranoia)')
     p.add_argument('--limit-per-task', type=int,
                    help='first N samples PER subset/category (evalscope --limit semantics); '
                         'composable with --limit (intersection)')
diff --git a/evalharness/eval/record.py b/evalharness/eval/record.py
index 62370cd..72edf98 100644
--- a/evalharness/eval/record.py
+++ b/evalharness/eval/record.py
@@ -56,7 +56,10 @@ class EvalReport(BaseModel):
     num_samples: int = 0
     num_failed_extractions: int = 0
     metrics: Dict[str, float] = Field(default_factory=dict)     # {'acc': 0.62}
-    metric_groups: Dict[str, Dict[str, float]] = Field(default_factory=dict)
+    # values may be None (perf stats the adapter couldn't measure), lists
+    # (repeats.scores) or nested dicts -- a strict float type rejected the
+    # file on LOAD and silently defeated report reuse
+    metric_groups: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
     # {'by_category': {'algebra': 0.7, ...}, 'pass_at_k': {'pass@1': .., 'pass@8': ..},
     #  'by_length_bin': {'8k': .., '32k': ..}}