HF endpoint auto-fallback: when huggingface.co is unreachable, probe mirrors (hf-mirror.com) and switch automatically with a one-line notice; error only if all fail

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/data/loader.py

@@ -462,22 +462,36 @@ def _load_from_hub(spec: DatasetSpec) -> List[Dict[str, Any]]:
     return [dict(r) for r in ds]
 
 
+_HF_MIRROR_FALLBACKS = ['https://hf-mirror.com']
+
+
 def _probe_hub_reachable(spec: DatasetSpec) -> None:
     """Fail fast when the HF endpoint is unreachable -- otherwise the hub
-    client retries silently for minutes and looks like a hang."""
+    client retries silently for minutes and looks like a hang.
+
+    When the DEFAULT endpoint is down but a mirror answers, switch to it
+    automatically (one-line notice, same behavior as --hf-endpoint)."""
     base = _hf_base()
-    try:
-        req = urllib.request.Request(base, method='HEAD',
-                                     headers={'User-Agent': 'evalharness/0.1'})
-        urllib.request.urlopen(req, timeout=8)
-        return
-    except Exception as e:
-        raise RuntimeError(
-            f'HuggingFace endpoint unreachable: {base}\n'
-            f'  reason: {type(e).__name__}: {str(e)[:80]}\n'
-            f'  dataset {spec.source!r} cannot download. Fix:\n'
-            f'  evalharness eval run ... --hf-endpoint https://hf-mirror.com\n'
-            f'  (or export HF_ENDPOINT=https://hf-mirror.com)') from e
+    err = None
+    for candidate in [base] + (_HF_MIRROR_FALLBACKS if base == 'https://huggingface.co' else []):
+        try:
+            req = urllib.request.Request(candidate, method='HEAD',
+                                         headers={'User-Agent': 'evalharness/0.1'})
+            urllib.request.urlopen(req, timeout=8)
+            if candidate != base:
+                os.environ['HF_ENDPOINT'] = candidate
+                print(f'· {base} unreachable -- falling back to {candidate}',
+                      flush=True)
+            return
+        except Exception as e:
+            err = e
+    raise RuntimeError(
+        f'HuggingFace endpoint unreachable: {base}'
+        + (f' (mirrors tried: {", ".join(_HF_MIRROR_FALLBACKS)})'
+           if base == 'https://huggingface.co' else '')
+        + f'\n  reason: {type(err).__name__}: {str(err)[:80]}\n'
+        f'  dataset {spec.source!r} cannot download. Fix:\n'
+        f'  evalharness eval run ... --hf-endpoint <a reachable endpoint>') from err
 
 
 def field_spec_to_record_fn(fields: FieldSpec) -> Callable[[Dict[str, Any]], Sample]: