diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 23eb5ec..16fbaa6 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -606,10 +606,16 @@ async def run_eval( fs_spec = dataclasses.replace(spec, split=fs_split) if spec is not None else None if fs_spec is not None: - from ..data.loader import load_raw_records + # go through the CACHED materialization path (not raw hub + # loads): a cached few-shot split never touches the + # network; first use downloads and caches it for offline + # runs afterwards + from ..data.dataset import Dataset fn = prov.resolve_record_fn() - fs_raw = load_raw_records(fs_spec) + fs_ds = Dataset(fs_spec, fn) + fs_ds.materialize() + fs_samples_all = list(fs_ds) # keep the WHOLE dev split when samples carry a category: # es selects domain-MATCHED exemplars per subject (mmlu # biology questions get biology exemplars), we do the same @@ -617,16 +623,16 @@ async def run_eval( def _lv_of(md): return (md or {}).get('category') or (md or {}).get('level') - cats = {_lv_of(fn(r).metadata) for r in fs_raw[:200]} + cats = {_lv_of(s.metadata) for s in fs_samples_all[:200]} style_is = getattr(spec, 'prompt_style', '') if spec is not None else '' if len(cats) > 1 and spec is not None and \ (style_is.startswith('cot_letter') or style_is == 'imo_es'): # mmlu-style per-subject OR math per-Level exemplars: # load the WHOLE few-shot split; assemble-time picks # domain-matched first-N (es reformat_subset semantics) - few_shot_samples = [fn(r) for r in fs_raw] + few_shot_samples = fs_samples_all else: - few_shot_samples = [fn(r) for r in fs_raw[:few_shot_num]] + few_shot_samples = fs_samples_all[:few_shot_num] except Exception as e: print(f'few-shot: could not load {fs_split} split ({type(e).__name__}: ' f'{str(e)[:80]}); continuing 0-shot', flush=True)