Scoring preflight: auto batch-pull for multi-image benches

Single-image benches keep fail-fast; >8 distinct images (swe's 500
per-instance) now pull concurrently with resume state inside the run
itself -- exactly like the old auto-pull behavior, just resilient:
scoring proceeds with whatever images landed, missing ones score 0
and only a total wipeout fails the bench. Ctrl+C-safe (state file),
network recovery resumes automatically on the next run.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/eval/runner.py

@@ -65,10 +65,24 @@ def evaluate(
             if getattr(s, 'sandbox', None) and s.sandbox.image:
                 _imgs.add(s.sandbox.image)
         if _imgs:
-            from ..sandbox.docker import ensure_image
+            # 多镜像 bench（swe 500 个逐题镜像）：并发批量拉取 + 断点 state，
+            # 拉到多少算多少、判分不中断（缺镜像的样本计 0 并在报告中标 missing）
+            # 少数镜像（bigcodebench 单镜像）保持 fail-fast
+            if len(_imgs) > 8:
+                from ..data.dataset import get_cache_root
+                from ..sandbox.pull_swe import pull_many
 
-            for _img in sorted(_imgs):
+                fails = pull_many(sorted(_imgs), str(get_cache_root()),
-                ensure_image(_img)
+                                  max_workers=4)
+                if fails == len(_imgs):
+                    raise RuntimeError(
+                        f'全部 {len(_imgs)} 个沙箱镜像拉取失败（网络/镜像源问'
+                        '题）；已保留断点，网络恢复后重跑自动续拉')
+            else:
+                from ..sandbox.docker import ensure_image
+
+                for _img in sorted(_imgs):
+                    ensure_image(_img)
     except RuntimeError:
         raise
     except Exception:

evalharness/sandbox/pull_swe.py

@@ -81,6 +81,45 @@ def _pull_one(image: str) -> tuple:
     return proc.returncode, '\n'.join(lines)
 
 
+def pull_many(names, cache_dir, max_workers: int = 4,
+              verbose: bool = True) -> int:
+    """批量拉取一组镜像名（断点续传复用 <cache-dir>/swe_pull_state.json）。
+
+    返回失败数。给判分预检用：多镜像 bench（swe 500 个）逐个 fail-fast
+    会把整个 bench 卡死在第一个失败上；这里并发拉、不中断、返回失败数。"""
+    cdir = Path(cache_dir)
+    state = _load_state(cdir)
+    local = set()
+    r = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'],
+                       capture_output=True, text=True)
+    if r.returncode == 0:
+        local = {l.strip() for l in r.stdout.splitlines() if l.strip()}
+    todo = [nm for nm in names
+            if nm not in state['done'] and nm not in local]
+    if verbose:
+        _log(f'· {len(names)} 个沙箱镜像: 本地/已完成 {len(names) - len(todo)}, '
+             f'待拉 {len(todo)} (并发 {max_workers}, 进度存 {_state_path(cdir)})')
+
+    def work(nm):
+        rc, out = _pull_one(nm)
+        if rc == 0:
+            state['done'].append(nm)
+        else:
+            state['failed'].append(nm)
+        _save_state(cdir, state)
+        return nm, rc
+
+    fails = 0
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        for i, (nm, rc) in enumerate(
+                pool.map(work, todo), 1):
+            if rc:
+                fails += 1
+            if verbose and (i % 10 == 0 or i == len(todo)):
+                _log(f'  镜像进度 [{i}/{len(todo)}] 失败 {fails}')
+    return fails
+
+
 def run_pull(dataset: str, cache_dir: str, max_workers: int = 4,
              dry_run: int = 0, retry_failed: bool = False):
     cdir = Path(cache_dir)