bigcodebench: official image + fail-fast preflight + no-retry on missing image

1140 real predictions scored 0.0% because the recipe referenced
'bigcodebench-sandbox:latest' -- a name nothing builds and docker.io
does not have; every sample then burned 3 pull-retries (~200s each).

- image -> bigcodebench/bigcodebench-evaluate:latest (the official hub
  image, same one evalscope uses)
- ensure_image() preflight in evaluate(): recipe-level AND sample-level
  images are verified/pulled ONCE before any container runs; missing ->
  seconds-fast bench failure with a fix hint instead of a silent 0.0%
- docker exec: 'Unable to find image'/'manifest unknown' class errors
  are permanent -- no 3x retry amplification

Verified with a bogus image: preflight raises in one pull-attempt with
the fix hint.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/eval/recipes/agent.py

@@ -56,7 +56,7 @@ def bigcodebench():
         extract='code_any',
         scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
                           # official sandbox image (bundles every task's deps)
-                          'image': 'bigcodebench-sandbox:latest',
+                          'image': 'bigcodebench/bigcodebench-evaluate:latest',  # official hub image, same as evalscope
                           'sandbox': 'docker', 'timeout_s': 120}},
         aggregators={'pass': 'pass_at_k'},
         exec_workers=12,

evalharness/eval/runner.py

@@ -51,6 +51,29 @@ def evaluate(
     aggregators = recipe.resolve_aggregators()
     ctx = ScoreContext(judge=judge, params={})
 
+    # fail-fast image preflight: recipe-level AND sample-level images are
+    # ensured (local or pulled once) BEFORE any container runs -- a missing
+    # image must kill the bench in seconds with a fix hint, not produce a
+    # 0.0% after hours of per-sample pull failures
+    try:
+        _imgs = set()
+        for spec in (recipe.scorers or {}).values():
+            p = spec if isinstance(spec, dict) else {}
+            if p.get('name') == 'execution' and p.get('sandbox') == 'docker'                     and p.get('image'):
+                _imgs.add(p['image'])
+        for s in samples[:200]:
+            if getattr(s, 'sandbox', None) and s.sandbox.image:
+                _imgs.add(s.sandbox.image)
+        if _imgs:
+            from ..sandbox.docker import ensure_image
+
+            for _img in sorted(_imgs):
+                ensure_image(_img)
+    except RuntimeError:
+        raise
+    except Exception:
+        pass  # no docker here (local sandbox): the scorer will complain
+
     # If any scorer executes in docker with per-sample images, overlap pulls
     # with scoring (run sample N while N+1..N+lookahead images download).
     bp = None

evalharness/sandbox/docker.py

@@ -26,6 +26,26 @@ def docker_available() -> bool:
     return _run(['docker', 'info']).returncode == 0
 
 
+def ensure_image(img: str) -> None:
+    """Fail-fast sandbox image preflight: present locally, else pull ONCE.
+
+    Without this, every sample's `docker run` tries its own pull at scoring
+    time -- a missing image burned 1140 x 3 retries x ~70s on bigcodebench
+    before anyone saw a 0.0%."""
+    if not img:
+        return
+    if _run(['docker', 'image', 'inspect', img]).returncode == 0:
+        return
+    r = _run(['docker', 'pull', img], timeout=1800)
+    if r.returncode != 0:
+        raise RuntimeError(
+            f'sandbox image {img!r} is not available: not local, and '
+            f'docker pull failed: {(r.stderr or "")[:200]}. '
+            'Fix: pull/build it first (for bigcodebench the official image '
+            'is bigcodebench/bigcodebench-evaluate:latest), then re-run with '
+            '--rescore to score the cached predictions.')
+
+
 @register_sandbox('docker')
 class DockerSandbox(Sandbox):
     name = 'docker'
@@ -90,6 +110,14 @@ class DockerSandbox(Sandbox):
                     raise
                 if proc.returncode != 125 or attempt == 2:
                     break
+                # image-not-found is PERMANENT: retrying it 3x per sample
+                # burned 1140 x ~200s on a nonexistent bigcodebench image
+                _nf = 'Unable to find image' in (proc.stderr or '') \
+                    or 'failed to resolve' in (proc.stderr or '') \
+                    or 'manifest unknown' in (proc.stderr or '') \
+                    or 'pull access denied' in (proc.stderr or '')
+                if _nf:
+                    break
                 # clear any husk; fresh name next try. Bounded: an rm against
                 # a bloated daemon can hang for minutes and silently eat the
                 # whole worker pool (7 of 8 workers were observed stuck here)