diff --git a/evalharness/data/datasets/swe_bench_verified.py b/evalharness/data/datasets/swe_bench_verified.py index 5415fd7..1d7263b 100644 --- a/evalharness/data/datasets/swe_bench_verified.py +++ b/evalharness/data/datasets/swe_bench_verified.py @@ -19,10 +19,17 @@ from ..spec import DatasetSpec def swe_bench_verified(): def to_sample(record: dict) -> Sample: instance_id = record['instance_id'] + # image naming: local swebench/ set uses {repo}_1776_{repo}-{num}; the + # newer official layout is sweb.eval.x86_64.{repo}__{repo}-{num} + img = f'sweb.eval.x86_64.{instance_id}' + if '__' in instance_id: + repo, num = instance_id.rsplit('-', 1) + r = repo.split('__')[0] + img = f'swebench/sweb.eval.x86_64.{r}_1776_{r.split("__")[-1]}-{num}' return Sample( input=record['problem_statement'], target=record['patch'], # gold patch (for oracle/oracle-check only) - sandbox=SandboxSpec(image=f'sweb.eval.x86_64.{instance_id}'), # official image naming + sandbox=SandboxSpec(image=img), metadata={ 'instance_id': instance_id, 'repo': record['repo'],