Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Report reuse: skip re-scoring when all predictions are checkpointed

Browse Source 
The checkpoint stores predictions, not scores -- reruns re-executed the
whole docker scoring pass even when nothing could change. When the
saved report matches (same model, same sample count) and every
prediction is checkpointed, reuse it; --rescore forces evaluation
(recipe/judge changed).

Also fixes EvalReport.load: metric_groups declared Dict[str, float]
rejected the None perf values (and the repeats scores list) the file
format actually contains -- every load raised ValidationError, which
is what silently killed reuse in testing.

Repeats>1 benches never reuse: their whole point is fresh sampling.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 06:12:30 +00:00
parent 97c4eef8fb
commit f272146b54
2 changed files with 36 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
evalharness/cli.py
View File

@ -728,7 +728,34 @@ def _cmd_eval_run(args) -> int:
_rep_secs = 0.0
_rep_tin = _rep_tout = 0
if model_spec: # generate + score in one go
report = None
# score cache: the checkpoint stores PREDICTIONS, not scores --
# but when every prediction is already checkpointed AND a saved
# report matches (same model, same sample count), re-scoring
# reproduces the same numbers, so reuse the report and skip the
# whole docker/exec scoring pass. --rescore forces evaluation
# (recipe/judge changed, or just paranoia).
if model_spec and out_dir and _repeats == 1 \
and not getattr(args, 'rescore', False):
from pathlib import Path as _P2
_rp = _P2(out_dir) / name / 'report.jsonl'
if _rp.exists():
try:
from evalharness.eval.record import EvalReport as _ER
_old = _ER.load(str(_rp))
_m_old = (_old.model or '').split('?')[-1]
_m_new = model_spec.split('?')[-1]
if _m_old == _m_new \
and _old.num_samples == (args.limit or sample_count):
report = _old
_emit('Reusing saved report -- predictions all '
'checkpointed (--rescore to re-evaluate)')
except Exception:
pass # unreadable/stale report: score normally
if report is None and model_spec: # generate + score in one go
from evalharness.model import run_eval
progress_reporter = None
@ -861,7 +888,7 @@ def _cmd_eval_run(args) -> int:
if _primary:
report.metrics[f'{_primary}_last_run'] = report.metrics[_primary]
report.metrics[_primary] = _mean
else:
elif report is None:
from evalharness.eval import evaluate
if not args.predictions:
@ -1166,6 +1193,9 @@ def build_parser() -> argparse.ArgumentParser:
p.add_argument('--resume', nargs='?', const=True, default=False,
help='resume from per-sample checkpoint (default path auto-derived; '
'pass a path to override)')
p.add_argument('--rescore', action='store_true',
help='force re-scoring even when a matching saved report '
'could be reused (change of recipe/judge, or paranoia)')
p.add_argument('--limit-per-task', type=int,
help='first N samples PER subset/category (evalscope --limit semantics); '
'composable with --limit (intersection)')

5
evalharness/eval/record.py
View File

@ -56,7 +56,10 @@ class EvalReport(BaseModel):
num_samples: int = 0
num_failed_extractions: int = 0
metrics: Dict[str, float] = Field(default_factory=dict) # {'acc': 0.62}
metric_groups: Dict[str, Dict[str, float]] = Field(default_factory=dict)
# values may be None (perf stats the adapter couldn't measure), lists
# (repeats.scores) or nested dicts -- a strict float type rejected the
# file on LOAD and silently defeated report reuse
metric_groups: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
# {'by_category': {'algebra': 0.7, ...}, 'pass_at_k': {'pass@1': .., 'pass@8': ..},
# 'by_length_bin': {'8k': .., '32k': ..}}