Optimize SWE-bench build_images: skip make_test_spec when images are pre-loaded
- make_test_spec() for all 500 samples was the real bottleneck that made swe_bench_verified appear stuck at 'Processing records: 0%' even when all instance images were already loaded locally. - Now image names are computed directly from instance IDs, matching the swebench naming convention. Local images are listed once and missing ones are reported immediately. make_test_spec is only called for images that actually need to be built. - With 500 pre-loaded images, build_images() now completes in ~45s instead of hanging indefinitely. Also add bash/diagnose_swe_images.py for quickly checking which images are missing locally.
This commit is contained in:
sora
parent
0a3b880da3
commit
ae5a1a192e
109
bash/diagnose_swe_images.py
Normal file
109
bash/diagnose_swe_images.py
Normal file
@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
诊断 swe_bench_verified 镜像准备阶段卡在哪里。
|
||||
|
||||
用法:
|
||||
python bash/diagnose_swe_images.py
|
||||
|
||||
它会模拟 evalscope 内部 build_images() 的前半段：
|
||||
1. 加载数据集
|
||||
2. 生成 500 个 instance 的镜像名
|
||||
3. 列出本地 Docker 镜像
|
||||
4. 对比哪些已存在、哪些缺失
|
||||
5. 打印每一步耗时
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# 让 evalscope 可导入
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.resolve()
|
||||
sys.path.insert(0, str(PROJECT_ROOT / 'evalscope'))
|
||||
|
||||
os.environ.setdefault('USE_MODELSCOPE_HUB', '1')
|
||||
os.environ.setdefault('HF_ENDPOINT', 'https://hf-mirror.com')
|
||||
os.environ.setdefault('PYTHONUNBUFFERED', '1')
|
||||
|
||||
from evalscope.api.dataset import RemoteDataLoader, FieldSpec
|
||||
from evalscope.benchmarks.swe_bench.utils import resolve_swebench_arch
|
||||
from swebench.harness.test_spec.test_spec import make_test_spec
|
||||
|
||||
|
||||
def get_local_images():
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f'ERROR: docker images failed: {result.stderr}')
|
||||
return set()
|
||||
return {line.strip() for line in result.stdout.splitlines() if line.strip()}
|
||||
|
||||
|
||||
def main():
|
||||
t0 = time.time()
|
||||
|
||||
print('==> 1. 加载 SWE-bench_Verified 数据集', flush=True)
|
||||
loader = RemoteDataLoader(
|
||||
data_id_or_path='princeton-nlp/SWE-bench_Verified',
|
||||
split='test',
|
||||
sample_fields=FieldSpec(
|
||||
input='problem_statement',
|
||||
metadata=[
|
||||
'instance_id', 'repo', 'base_commit', 'patch',
|
||||
'PASS_TO_PASS', 'FAIL_TO_PASS', 'test_patch', 'version',
|
||||
'environment_setup_commit', 'hints_text', 'created_at',
|
||||
],
|
||||
),
|
||||
)
|
||||
samples = loader.load()
|
||||
print(f' 样本数: {len(samples)}, 耗时: {time.time()-t0:.2f}s', flush=True)
|
||||
|
||||
t1 = time.time()
|
||||
print('==> 2. 生成 500 个镜像名 (make_test_spec)', flush=True)
|
||||
missing_repo = 0
|
||||
image_names = []
|
||||
last_log = time.time()
|
||||
for idx, s in enumerate(samples):
|
||||
metadata = s.metadata if hasattr(s, 'metadata') else s
|
||||
instance_id = metadata.get('instance_id')
|
||||
try:
|
||||
arch = resolve_swebench_arch(instance_id, '')
|
||||
spec = make_test_spec(metadata, namespace='swebench', arch=arch)
|
||||
image_names.append(spec.instance_image_key)
|
||||
except KeyError as e:
|
||||
missing_repo += 1
|
||||
if missing_repo <= 3:
|
||||
print(f' WARNING: instance {instance_id} 缺少字段: {e}', flush=True)
|
||||
if time.time() - last_log > 10:
|
||||
print(f' 进度: {idx+1}/{len(samples)} ...', flush=True)
|
||||
last_log = time.time()
|
||||
print(f' 生成镜像名: {len(image_names)}, 缺失字段: {missing_repo}, 耗时: {time.time()-t1:.2f}s', flush=True)
|
||||
|
||||
t2 = time.time()
|
||||
print('==> 3. 扫描本地 Docker 镜像', flush=True)
|
||||
local_images = get_local_images()
|
||||
print(f' 本地镜像总数: {len(local_images)}, 耗时: {time.time()-t2:.2f}s', flush=True)
|
||||
|
||||
t3 = time.time()
|
||||
print('==> 4. 对比缺失镜像', flush=True)
|
||||
needed = set(image_names)
|
||||
present = needed & local_images
|
||||
missing = needed - local_images
|
||||
print(f' 需要: {len(needed)}, 已存在: {len(present)}, 缺失: {len(missing)}, 耗时: {time.time()-t3:.2f}s', flush=True)
|
||||
|
||||
if missing:
|
||||
print('\n前 10 个缺失镜像:', flush=True)
|
||||
for name in list(missing)[:10]:
|
||||
print(f' {name}', flush=True)
|
||||
else:
|
||||
print('\n所有需要镜像均已存在，evalscope 不应该再拉取/构建。', flush=True)
|
||||
|
||||
print(f'\n总耗时: {time.time()-t0:.2f}s', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -57,18 +57,43 @@ def build_images(
|
||||
# Note that remote images are named eg "sphinx-doc_1776_sphinx-11502"
|
||||
namespace = (dockerhub_username or 'swebench') if use_remote_images else None
|
||||
|
||||
def _get_instance_image_name(instance_id: str) -> str:
|
||||
"""Fast image name computation that matches make_test_spec output."""
|
||||
arch = resolve_swebench_arch(instance_id, force_arch)
|
||||
if namespace:
|
||||
# Remote / pre-built image format: swebench/sweb.eval.x86_64.repo_1776_name-123:latest
|
||||
updated_id = instance_id.replace('__', '_1776_')
|
||||
return f'{namespace}/sweb.eval.{arch}.{updated_id}:latest'
|
||||
# Local build format: sweb.eval.x86_64.repo__name-123:latest
|
||||
return f'sweb.eval.{arch}.{instance_id}:latest'
|
||||
|
||||
# Fast path: compute image names directly without calling make_test_spec.
|
||||
# make_test_spec is very slow for 500 samples and is only needed when we
|
||||
# actually have to build missing images locally.
|
||||
logger.info('Computing SWE-Bench image names from instance IDs...')
|
||||
for swebench_instance in samples_hf:
|
||||
arch = resolve_swebench_arch(swebench_instance['instance_id'], force_arch)
|
||||
test_spec = make_test_spec(swebench_instance, namespace=namespace, arch=arch)
|
||||
docker_image_name = test_spec.instance_image_key
|
||||
id_to_docker_image[swebench_instance['instance_id']] = docker_image_name
|
||||
id_to_test_spec[swebench_instance['instance_id']] = test_spec
|
||||
instance_id = swebench_instance['instance_id']
|
||||
id_to_docker_image[instance_id] = _get_instance_image_name(instance_id)
|
||||
|
||||
# Get list of locally available Docker images
|
||||
logger.info('Listing local Docker images...')
|
||||
available_docker_images = _get_available_docker_images()
|
||||
samples_to_build_images_for = [
|
||||
s for s in samples_hf if id_to_docker_image[s['instance_id']] not in available_docker_images
|
||||
]
|
||||
logger.info(
|
||||
f'SWE-Bench images: {len(samples_hf)} needed, '
|
||||
f'{len(samples_hf) - len(samples_to_build_images_for)} already local, '
|
||||
f'{len(samples_to_build_images_for)} missing'
|
||||
)
|
||||
|
||||
# Only build expensive test_specs for images that are actually missing.
|
||||
if samples_to_build_images_for:
|
||||
logger.info('Building test specs for missing images (this may take a while)...')
|
||||
for swebench_instance in samples_to_build_images_for:
|
||||
arch = resolve_swebench_arch(swebench_instance['instance_id'], force_arch)
|
||||
test_spec = make_test_spec(swebench_instance, namespace=namespace, arch=arch)
|
||||
id_to_test_spec[swebench_instance['instance_id']] = test_spec
|
||||
|
||||
# Try to pull images from Docker Hub first if requested
|
||||
if use_remote_images and len(samples_to_build_images_for) > 0:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user