diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py index dde919f..681c053 100644 --- a/evalharness/data/loader.py +++ b/evalharness/data/loader.py @@ -422,11 +422,32 @@ def _hf_download(repo: str, path: str, dest_dir: Path) -> Path: def _load_from_hf_raw(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[Dict[str, Any]]: import hashlib + # the mirror's tree API degrades at night and returns TRUNCATED listings + # (a repo with 9 parquets listed as [.gitattributes, README.md]); the + # blobs from a previous run are already in the shared store -- use them + blob_dir = get_cache_root() / '.raw' / hashlib.md5(spec.source.encode()).hexdigest()[:10] + + def _cached_blobs() -> List[str]: + if not blob_dir.exists(): + return [] + have = [f.name for f in blob_dir.iterdir() + if os.path.splitext(f)[1] in _SUPPORTED_EXTS] + pref = [f for f in have if f.startswith(spec.subset)] or have + return pref + files = _hf_list_files(spec.source) - if not files: - raise FileNotFoundError(f'no files found on HF dataset {spec.source!r}') - selected = _hf_match_files(spec, files) + selected = _hf_match_files(spec, files) if files else [] if not selected: + cached = _cached_blobs() + if cached: + print(f'ยท listing unavailable/degraded -- using {len(cached)} ' + f'cached file(s) from {blob_dir}', flush=True) + records: List[Dict[str, Any]] = [] + for f in sorted(cached): + records.extend(_read_file(str(blob_dir / f))) + if raw_dir is not None: + _link_or_copy_all([blob_dir / f for f in sorted(cached)], raw_dir) + return records raise FileNotFoundError( f'no data file for subset={spec.subset!r} split={spec.split!r} in ' f'HF {spec.source!r}. Available (first 10): {files[:10]}'