diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index ba96821..6911411 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -65,10 +65,24 @@ def evaluate( if getattr(s, 'sandbox', None) and s.sandbox.image: _imgs.add(s.sandbox.image) if _imgs: - from ..sandbox.docker import ensure_image + # 多镜像 bench(swe 500 个逐题镜像):并发批量拉取 + 断点 state, + # 拉到多少算多少、判分不中断(缺镜像的样本计 0 并在报告中标 missing) + # 少数镜像(bigcodebench 单镜像)保持 fail-fast + if len(_imgs) > 8: + from ..data.dataset import get_cache_root + from ..sandbox.pull_swe import pull_many - for _img in sorted(_imgs): - ensure_image(_img) + fails = pull_many(sorted(_imgs), str(get_cache_root()), + max_workers=4) + if fails == len(_imgs): + raise RuntimeError( + f'全部 {len(_imgs)} 个沙箱镜像拉取失败(网络/镜像源问' + '题);已保留断点,网络恢复后重跑自动续拉') + else: + from ..sandbox.docker import ensure_image + + for _img in sorted(_imgs): + ensure_image(_img) except RuntimeError: raise except Exception: diff --git a/evalharness/sandbox/pull_swe.py b/evalharness/sandbox/pull_swe.py index d17034e..d2fcb5f 100644 --- a/evalharness/sandbox/pull_swe.py +++ b/evalharness/sandbox/pull_swe.py @@ -81,6 +81,45 @@ def _pull_one(image: str) -> tuple: return proc.returncode, '\n'.join(lines) +def pull_many(names, cache_dir, max_workers: int = 4, + verbose: bool = True) -> int: + """批量拉取一组镜像名(断点续传复用 /swe_pull_state.json)。 + + 返回失败数。给判分预检用:多镜像 bench(swe 500 个)逐个 fail-fast + 会把整个 bench 卡死在第一个失败上;这里并发拉、不中断、返回失败数。""" + cdir = Path(cache_dir) + state = _load_state(cdir) + local = set() + r = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'], + capture_output=True, text=True) + if r.returncode == 0: + local = {l.strip() for l in r.stdout.splitlines() if l.strip()} + todo = [nm for nm in names + if nm not in state['done'] and nm not in local] + if verbose: + _log(f'· {len(names)} 个沙箱镜像: 本地/已完成 {len(names) - len(todo)}, ' + f'待拉 {len(todo)} (并发 {max_workers}, 进度存 {_state_path(cdir)})') + + def work(nm): + rc, out = _pull_one(nm) + if rc == 0: + state['done'].append(nm) + else: + state['failed'].append(nm) + _save_state(cdir, state) + return nm, rc + + fails = 0 + with ThreadPoolExecutor(max_workers=max_workers) as pool: + for i, (nm, rc) in enumerate( + pool.map(work, todo), 1): + if rc: + fails += 1 + if verbose and (i % 10 == 0 or i == len(todo)): + _log(f' 镜像进度 [{i}/{len(todo)}] 失败 {fails}') + return fails + + def run_pull(dataset: str, cache_dir: str, max_workers: int = 4, dry_run: int = 0, retry_failed: bool = False): cdir = Path(cache_dir)