Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HF hub downloads: reachability probe fails fast with the mirror fix hint (was minutes of silent retries looking like a hang); status lines around hub downloads (start + records + elapsed)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:51:09 +00:00
parent afe2fb8d28
commit c728db5f35
1 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
evalharness/data/loader.py
View File

@ -14,6 +14,7 @@ import csv
import json import json
import os import os
import sys import sys
import time
import re import re
import shutil import shutil
import urllib.request import urllib.request
@ -449,14 +450,36 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]:
f'dataset {spec.name!r} lives on a hub ({spec.source!r}); ' f'dataset {spec.name!r} lives on a hub ({spec.source!r}); '
'install it first: pip install evalharness (light deps are default)' 'install it first: pip install evalharness (light deps are default)'
) )
_probe_hub_reachable(spec)
kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS} kwargs = {k: v for k, v in spec.params.items() if k not in _RESERVED_PARAMS}
# filter_column subsets select rows by column value post-load, so the hub # filter_column subsets select rows by column value post-load, so the hub
# load itself must always use the default config. # load itself must always use the default config.
subset = None if (spec.subset == 'default' or spec.params.get('filter_column')) else spec.subset subset = None if (spec.subset == 'default' or spec.params.get('filter_column')) else spec.subset
print(f'· downloading {spec.source} from HF hub ({_hf_base()}) ...', flush=True)
t0 = time.monotonic()
ds = datasets.load_dataset(spec.source, subset, split=spec.split, revision=spec.version, **kwargs) ds = datasets.load_dataset(spec.source, subset, split=spec.split, revision=spec.version, **kwargs)
print(f'· downloaded + parsed {len(ds)} records in {time.monotonic() - t0:.0f}s', flush=True)
return [dict(r) for r in ds] return [dict(r) for r in ds]
def _probe_hub_reachable(spec: DatasetSpec) -> None:
"""Fail fast when the HF endpoint is unreachable -- otherwise the hub
client retries silently for minutes and looks like a hang."""
base = _hf_base()
try:
req = urllib.request.Request(base, method='HEAD',
headers={'User-Agent': 'evalharness/0.1'})
urllib.request.urlopen(req, timeout=8)
return
except Exception as e:
raise RuntimeError(
f'HuggingFace endpoint unreachable: {base}\n'
f' reason: {type(e).__name__}: {str(e)[:80]}\n'
f' dataset {spec.source!r} cannot download. Fix:\n'
f' evalharness eval run ... --hf-endpoint https://hf-mirror.com\n'
f' (or export HF_ENDPOINT=https://hf-mirror.com)') from e
def field_spec_to_record_fn(fields: FieldSpec) -> Callable[[Dict[str, Any]], Sample]: def field_spec_to_record_fn(fields: FieldSpec) -> Callable[[Dict[str, Any]], Sample]:
"""Build the default record->Sample converter from a FieldSpec.""" """Build the default record->Sample converter from a FieldSpec."""