diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index ed6002c..9e0993f 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -56,7 +56,7 @@ def bigcodebench(): extract='code_any', scorers={'pass': {'name': 'execution', 'harness': _bcb_harness, # official sandbox image (bundles every task's deps) - 'image': 'bigcodebench-sandbox:latest', + 'image': 'bigcodebench/bigcodebench-evaluate:latest', # official hub image, same as evalscope 'sandbox': 'docker', 'timeout_s': 120}}, aggregators={'pass': 'pass_at_k'}, exec_workers=12, diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index 323925f..97da2ba 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -51,6 +51,29 @@ def evaluate( aggregators = recipe.resolve_aggregators() ctx = ScoreContext(judge=judge, params={}) + # fail-fast image preflight: recipe-level AND sample-level images are + # ensured (local or pulled once) BEFORE any container runs -- a missing + # image must kill the bench in seconds with a fix hint, not produce a + # 0.0% after hours of per-sample pull failures + try: + _imgs = set() + for spec in (recipe.scorers or {}).values(): + p = spec if isinstance(spec, dict) else {} + if p.get('name') == 'execution' and p.get('sandbox') == 'docker' and p.get('image'): + _imgs.add(p['image']) + for s in samples[:200]: + if getattr(s, 'sandbox', None) and s.sandbox.image: + _imgs.add(s.sandbox.image) + if _imgs: + from ..sandbox.docker import ensure_image + + for _img in sorted(_imgs): + ensure_image(_img) + except RuntimeError: + raise + except Exception: + pass # no docker here (local sandbox): the scorer will complain + # If any scorer executes in docker with per-sample images, overlap pulls # with scoring (run sample N while N+1..N+lookahead images download). bp = None diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py index 08cb35e..860dd8d 100644 --- a/evalharness/sandbox/docker.py +++ b/evalharness/sandbox/docker.py @@ -26,6 +26,26 @@ def docker_available() -> bool: return _run(['docker', 'info']).returncode == 0 +def ensure_image(img: str) -> None: + """Fail-fast sandbox image preflight: present locally, else pull ONCE. + + Without this, every sample's `docker run` tries its own pull at scoring + time -- a missing image burned 1140 x 3 retries x ~70s on bigcodebench + before anyone saw a 0.0%.""" + if not img: + return + if _run(['docker', 'image', 'inspect', img]).returncode == 0: + return + r = _run(['docker', 'pull', img], timeout=1800) + if r.returncode != 0: + raise RuntimeError( + f'sandbox image {img!r} is not available: not local, and ' + f'docker pull failed: {(r.stderr or "")[:200]}. ' + 'Fix: pull/build it first (for bigcodebench the official image ' + 'is bigcodebench/bigcodebench-evaluate:latest), then re-run with ' + '--rescore to score the cached predictions.') + + @register_sandbox('docker') class DockerSandbox(Sandbox): name = 'docker' @@ -90,6 +110,14 @@ class DockerSandbox(Sandbox): raise if proc.returncode != 125 or attempt == 2: break + # image-not-found is PERMANENT: retrying it 3x per sample + # burned 1140 x ~200s on a nonexistent bigcodebench image + _nf = 'Unable to find image' in (proc.stderr or '') \ + or 'failed to resolve' in (proc.stderr or '') \ + or 'manifest unknown' in (proc.stderr or '') \ + or 'pull access denied' in (proc.stderr or '') + if _nf: + break # clear any husk; fresh name next try. Bounded: an rm against # a bloated daemon can hang for minutes and silently eat the # whole worker pool (7 of 8 workers were observed stuck here)