Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix checkpoint cross-contamination (scope by dataset name from run_eval, not inferred); ladder20 comparison artifacts

Browse Source
This commit is contained in:
sora 2026-08-26 02:24:00 +00:00
parent 78459c974e
commit f3514d7c08
1 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
evalharness/model/runner.py
View File

@ -38,6 +38,7 @@ async def generate_predictions(
attach_context_keys: tuple = ('passage', 'context'),
limit_per_task: Optional[int] = None,
checkpoint: Union[bool, str] = False,
dataset_name: str = 'adhoc',
few_shot_num: int = 0,
few_shot_samples: Optional[List[Sample]] = None,
few_shot_text: Optional[str] = None,
@ -169,13 +170,11 @@ async def generate_predictions(
if checkpoint:
from ..eval.checkpoint import CheckpointStore, checkpoint_path
ckpt = checkpoint if isinstance(checkpoint, str) else None
if ckpt is None:
from ..eval.checkpoint import checkpoint_path as _cp
ckpt = _cp('/data/evalharness' if False else os.path.expanduser('~/.cache/evalharness'),
getattr(spec, 'name', 'adhoc') if spec is not None else 'adhoc',
model_spec)
if isinstance(checkpoint, str):
ckpt = checkpoint
else:
ckpt = checkpoint_path(os.path.expanduser('~/.cache/evalharness'),
dataset_name, adapter.model or str(adapter))
ckpt_store = CheckpointStore(ckpt, model=adapter.model or str(adapter))
restored = ckpt_store.load()
else:
@ -254,6 +253,7 @@ async def run_eval(
max_input_chars: int = 0,
limit_per_task: Optional[int] = None,
checkpoint: Union[bool, str] = False,
dataset_name: str = 'adhoc',
few_shot_num: int = -1,
prompt_style: str = 'strict_letter',
) -> EvalReport:
@ -336,6 +336,7 @@ async def run_eval(
system=system, max_turns=max_turns, max_input_chars=max_input_chars,
limit_per_task=limit_per_task,
checkpoint=checkpoint,
dataset_name=name,
few_shot_num=few_shot_num,
few_shot_samples=few_shot_samples, few_shot_text=few_shot_text,
prompt_style=prompt_style)