Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix checkpoint cross-contamination (scope by dataset name from run_eval, not inferred); ladder20 comparison artifacts

Browse Source
This commit is contained in:
sora 2026-08-26 02:24:00 +00:00
parent 78459c974e
commit f3514d7c08
1 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
evalharness/model/runner.py
View File

@ -38,6 +38,7 @@ async def generate_predictions(
attach_context_keys: tuple = ('passage', 'context'), attach_context_keys: tuple = ('passage', 'context'),
limit_per_task: Optional[int] = None, limit_per_task: Optional[int] = None,
checkpoint: Union[bool, str] = False, checkpoint: Union[bool, str] = False,
dataset_name: str = 'adhoc',
few_shot_num: int = 0, few_shot_num: int = 0,
few_shot_samples: Optional[List[Sample]] = None, few_shot_samples: Optional[List[Sample]] = None,
few_shot_text: Optional[str] = None, few_shot_text: Optional[str] = None,
@ -169,13 +170,11 @@ async def generate_predictions(
if checkpoint: if checkpoint:
from ..eval.checkpoint import CheckpointStore, checkpoint_path from ..eval.checkpoint import CheckpointStore, checkpoint_path
ckpt = checkpoint if isinstance(checkpoint, str) else None if isinstance(checkpoint, str):
if ckpt is None: ckpt = checkpoint
from ..eval.checkpoint import checkpoint_path as _cp else:
ckpt = checkpoint_path(os.path.expanduser('~/.cache/evalharness'),
ckpt = _cp('/data/evalharness' if False else os.path.expanduser('~/.cache/evalharness'), dataset_name, adapter.model or str(adapter))
getattr(spec, 'name', 'adhoc') if spec is not None else 'adhoc',
model_spec)
ckpt_store = CheckpointStore(ckpt, model=adapter.model or str(adapter)) ckpt_store = CheckpointStore(ckpt, model=adapter.model or str(adapter))
restored = ckpt_store.load() restored = ckpt_store.load()
else: else:
@ -254,6 +253,7 @@ async def run_eval(
max_input_chars: int = 0, max_input_chars: int = 0,
limit_per_task: Optional[int] = None, limit_per_task: Optional[int] = None,
checkpoint: Union[bool, str] = False, checkpoint: Union[bool, str] = False,
dataset_name: str = 'adhoc',
few_shot_num: int = -1, few_shot_num: int = -1,
prompt_style: str = 'strict_letter', prompt_style: str = 'strict_letter',
) -> EvalReport: ) -> EvalReport:
@ -336,6 +336,7 @@ async def run_eval(
system=system, max_turns=max_turns, max_input_chars=max_input_chars, system=system, max_turns=max_turns, max_input_chars=max_input_chars,
limit_per_task=limit_per_task, limit_per_task=limit_per_task,
checkpoint=checkpoint, checkpoint=checkpoint,
dataset_name=name,
few_shot_num=few_shot_num, few_shot_num=few_shot_num,
few_shot_samples=few_shot_samples, few_shot_text=few_shot_text, few_shot_samples=few_shot_samples, few_shot_text=few_shot_text,
prompt_style=prompt_style) prompt_style=prompt_style)