From ae5a1a192ee8b2eb25553d881d0b1af242870e8b Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 27 Jul 2026 08:46:15 +0000 Subject: [PATCH] Optimize SWE-bench build_images: skip make_test_spec when images are pre-loaded - make_test_spec() for all 500 samples was the real bottleneck that made swe_bench_verified appear stuck at 'Processing records: 0%' even when all instance images were already loaded locally. - Now image names are computed directly from instance IDs, matching the swebench naming convention. Local images are listed once and missing ones are reported immediately. make_test_spec is only called for images that actually need to be built. - With 500 pre-loaded images, build_images() now completes in ~45s instead of hanging indefinitely. Also add bash/diagnose_swe_images.py for quickly checking which images are missing locally. --- bash/diagnose_swe_images.py | 109 ++++++++++++++++++ .../benchmarks/swe_bench/build_images.py | 35 +++++- 2 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 bash/diagnose_swe_images.py diff --git a/bash/diagnose_swe_images.py b/bash/diagnose_swe_images.py new file mode 100644 index 0000000..a664587 --- /dev/null +++ b/bash/diagnose_swe_images.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +诊断 swe_bench_verified 镜像准备阶段卡在哪里。 + +用法: + python bash/diagnose_swe_images.py + +它会模拟 evalscope 内部 build_images() 的前半段： +1. 加载数据集 +2. 生成 500 个 instance 的镜像名 +3. 列出本地 Docker 镜像 +4. 对比哪些已存在、哪些缺失 +5. 打印每一步耗时 +""" + +import os +import sys +import time +from pathlib import Path + +# 让 evalscope 可导入 +PROJECT_ROOT = Path(__file__).parent.parent.resolve() +sys.path.insert(0, str(PROJECT_ROOT / 'evalscope')) + +os.environ.setdefault('USE_MODELSCOPE_HUB', '1') +os.environ.setdefault('HF_ENDPOINT', 'https://hf-mirror.com') +os.environ.setdefault('PYTHONUNBUFFERED', '1') + +from evalscope.api.dataset import RemoteDataLoader, FieldSpec +from evalscope.benchmarks.swe_bench.utils import resolve_swebench_arch +from swebench.harness.test_spec.test_spec import make_test_spec + + +def get_local_images(): + import subprocess + result = subprocess.run( + ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'], + capture_output=True, text=True + ) + if result.returncode != 0: + print(f'ERROR: docker images failed: {result.stderr}') + return set() + return {line.strip() for line in result.stdout.splitlines() if line.strip()} + + +def main(): + t0 = time.time() + + print('==> 1. 加载 SWE-bench_Verified 数据集', flush=True) + loader = RemoteDataLoader( + data_id_or_path='princeton-nlp/SWE-bench_Verified', + split='test', + sample_fields=FieldSpec( + input='problem_statement', + metadata=[ + 'instance_id', 'repo', 'base_commit', 'patch', + 'PASS_TO_PASS', 'FAIL_TO_PASS', 'test_patch', 'version', + 'environment_setup_commit', 'hints_text', 'created_at', + ], + ), + ) + samples = loader.load() + print(f' 样本数: {len(samples)}, 耗时: {time.time()-t0:.2f}s', flush=True) + + t1 = time.time() + print('==> 2. 生成 500 个镜像名 (make_test_spec)', flush=True) + missing_repo = 0 + image_names = [] + last_log = time.time() + for idx, s in enumerate(samples): + metadata = s.metadata if hasattr(s, 'metadata') else s + instance_id = metadata.get('instance_id') + try: + arch = resolve_swebench_arch(instance_id, '') + spec = make_test_spec(metadata, namespace='swebench', arch=arch) + image_names.append(spec.instance_image_key) + except KeyError as e: + missing_repo += 1 + if missing_repo <= 3: + print(f' WARNING: instance {instance_id} 缺少字段: {e}', flush=True) + if time.time() - last_log > 10: + print(f' 进度: {idx+1}/{len(samples)} ...', flush=True) + last_log = time.time() + print(f' 生成镜像名: {len(image_names)}, 缺失字段: {missing_repo}, 耗时: {time.time()-t1:.2f}s', flush=True) + + t2 = time.time() + print('==> 3. 扫描本地 Docker 镜像', flush=True) + local_images = get_local_images() + print(f' 本地镜像总数: {len(local_images)}, 耗时: {time.time()-t2:.2f}s', flush=True) + + t3 = time.time() + print('==> 4. 对比缺失镜像', flush=True) + needed = set(image_names) + present = needed & local_images + missing = needed - local_images + print(f' 需要: {len(needed)}, 已存在: {len(present)}, 缺失: {len(missing)}, 耗时: {time.time()-t3:.2f}s', flush=True) + + if missing: + print('\n前 10 个缺失镜像:', flush=True) + for name in list(missing)[:10]: + print(f' {name}', flush=True) + else: + print('\n所有需要镜像均已存在，evalscope 不应该再拉取/构建。', flush=True) + + print(f'\n总耗时: {time.time()-t0:.2f}s', flush=True) + + +if __name__ == '__main__': + main() diff --git a/evalscope/evalscope/benchmarks/swe_bench/build_images.py b/evalscope/evalscope/benchmarks/swe_bench/build_images.py index 302413b..7bb1f90 100644 --- a/evalscope/evalscope/benchmarks/swe_bench/build_images.py +++ b/evalscope/evalscope/benchmarks/swe_bench/build_images.py @@ -57,18 +57,43 @@ def build_images( # Note that remote images are named eg "sphinx-doc_1776_sphinx-11502" namespace = (dockerhub_username or 'swebench') if use_remote_images else None + def _get_instance_image_name(instance_id: str) -> str: + """Fast image name computation that matches make_test_spec output.""" + arch = resolve_swebench_arch(instance_id, force_arch) + if namespace: + # Remote / pre-built image format: swebench/sweb.eval.x86_64.repo_1776_name-123:latest + updated_id = instance_id.replace('__', '_1776_') + return f'{namespace}/sweb.eval.{arch}.{updated_id}:latest' + # Local build format: sweb.eval.x86_64.repo__name-123:latest + return f'sweb.eval.{arch}.{instance_id}:latest' + + # Fast path: compute image names directly without calling make_test_spec. + # make_test_spec is very slow for 500 samples and is only needed when we + # actually have to build missing images locally. + logger.info('Computing SWE-Bench image names from instance IDs...') for swebench_instance in samples_hf: - arch = resolve_swebench_arch(swebench_instance['instance_id'], force_arch) - test_spec = make_test_spec(swebench_instance, namespace=namespace, arch=arch) - docker_image_name = test_spec.instance_image_key - id_to_docker_image[swebench_instance['instance_id']] = docker_image_name - id_to_test_spec[swebench_instance['instance_id']] = test_spec + instance_id = swebench_instance['instance_id'] + id_to_docker_image[instance_id] = _get_instance_image_name(instance_id) # Get list of locally available Docker images + logger.info('Listing local Docker images...') available_docker_images = _get_available_docker_images() samples_to_build_images_for = [ s for s in samples_hf if id_to_docker_image[s['instance_id']] not in available_docker_images ] + logger.info( + f'SWE-Bench images: {len(samples_hf)} needed, ' + f'{len(samples_hf) - len(samples_to_build_images_for)} already local, ' + f'{len(samples_to_build_images_for)} missing' + ) + + # Only build expensive test_specs for images that are actually missing. + if samples_to_build_images_for: + logger.info('Building test specs for missing images (this may take a while)...') + for swebench_instance in samples_to_build_images_for: + arch = resolve_swebench_arch(swebench_instance['instance_id'], force_arch) + test_spec = make_test_spec(swebench_instance, namespace=namespace, arch=arch) + id_to_test_spec[swebench_instance['instance_id']] = test_spec # Try to pull images from Docker Hub first if requested if use_remote_images and len(samples_to_build_images_for) > 0: