f662006517
Ports es's swe_bench_agentic_adapter into our plugin architecture: - env swe_agentic: per-sample LONG-RUNNING container (official sweb image, /testbed, bash -lc like the testbed startup files expect), single bash tool via function calling, sentinel-submission protocol (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT + patch), git-diff fallback; observations capped at 30k chars - dataset swe_bench_verified_agentic: same princeton source/converter, separate bench name so both variants coexist - recipe: recovered patch + OFFICIAL test_patch applied in-container, FAIL_TO_PASS + capped PASS_TO_PASS via conda testbed pytest, 1800s - config: max_turns 250, env swe_agentic Single-turn swe_bench_verified is untouched. Co-Authored-By: Claude <noreply@anthropic.com>
44 lines
1.7 KiB
Python
44 lines
1.7 KiB
Python
"""SWE-bench Verified (official source: princeton-nlp/SWE-bench_Verified)."""
|
|
|
|
from ..sample import Sample, SandboxSpec
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='swe_bench_verified',
|
|
source='princeton-nlp/SWE-bench_Verified', # official: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
|
|
split='test',
|
|
gen_config={'temperature': 0.0, 'max_tokens': 32768},
|
|
task_type='agent',
|
|
tags=['code', 'agent', 'swe'],
|
|
requires=['docker'],
|
|
description='SWE-bench Verified; per-instance docker image carried in Sample.sandbox.',
|
|
)
|
|
)
|
|
def _record_to_sample(record: dict) -> Sample:
|
|
instance_id = record['instance_id']
|
|
# EXACT Docker Hub naming (verified by es's pull state: 500/500):
|
|
# swebench/sweb.eval.x86_64.{instance_id.lower() with __->_1776_}:latest
|
|
# -- per-instance images ARE published under the swebench/ namespace
|
|
img = ('swebench/sweb.eval.x86_64.'
|
|
+ instance_id.lower().replace('__', '_1776_') + ':latest')
|
|
return Sample(
|
|
input=record['problem_statement'],
|
|
target=record['patch'], # gold patch (for oracle/oracle-check only)
|
|
sandbox=SandboxSpec(image=img),
|
|
metadata={
|
|
'instance_id': instance_id,
|
|
'repo': record['repo'],
|
|
'base_commit': record['base_commit'],
|
|
'test_patch': record['test_patch'],
|
|
'FAIL_TO_PASS': record['FAIL_TO_PASS'],
|
|
'PASS_TO_PASS': record['PASS_TO_PASS'],
|
|
'environment_setup_commit': record.get('environment_setup_commit'),
|
|
'difficulty': record.get('difficulty'),
|
|
},
|
|
)
|
|
|
|
return to_sample
|