From 0e1e31ba109a7d07a5167582a47d3ddd193ecb86 Mon Sep 17 00:00:00 2001 From: sora Date: Wed, 26 Aug 2026 06:40:55 +0000 Subject: [PATCH] swe_bench_verified end-to-end verified (oracle patch -> official sweb image -> resolved); image naming mapped to local swebench/ set; ladder2 complete 28/28 --- evalharness/data/datasets/swe_bench_verified.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evalharness/data/datasets/swe_bench_verified.py b/evalharness/data/datasets/swe_bench_verified.py index 5415fd7..1d7263b 100644 --- a/evalharness/data/datasets/swe_bench_verified.py +++ b/evalharness/data/datasets/swe_bench_verified.py @@ -19,10 +19,17 @@ from ..spec import DatasetSpec def swe_bench_verified(): def to_sample(record: dict) -> Sample: instance_id = record['instance_id'] + # image naming: local swebench/ set uses {repo}_1776_{repo}-{num}; the + # newer official layout is sweb.eval.x86_64.{repo}__{repo}-{num} + img = f'sweb.eval.x86_64.{instance_id}' + if '__' in instance_id: + repo, num = instance_id.rsplit('-', 1) + r = repo.split('__')[0] + img = f'swebench/sweb.eval.x86_64.{r}_1776_{r.split("__")[-1]}-{num}' return Sample( input=record['problem_statement'], target=record['patch'], # gold patch (for oracle/oracle-check only) - sandbox=SandboxSpec(image=f'sweb.eval.x86_64.{instance_id}'), # official image naming + sandbox=SandboxSpec(image=img), metadata={ 'instance_id': instance_id, 'repo': record['repo'],