From f66200651772c9781f5d27a97c7cf4222c3550bd Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 18 Sep 2026 02:29:35 +0000 Subject: [PATCH] swe_bench_verified_agentic: multi-turn SWE agent (mini-swe-agent protocol) Ports es's swe_bench_agentic_adapter into our plugin architecture: - env swe_agentic: per-sample LONG-RUNNING container (official sweb image, /testbed, bash -lc like the testbed startup files expect), single bash tool via function calling, sentinel-submission protocol (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT + patch), git-diff fallback; observations capped at 30k chars - dataset swe_bench_verified_agentic: same princeton source/converter, separate bench name so both variants coexist - recipe: recovered patch + OFFICIAL test_patch applied in-container, FAIL_TO_PASS + capped PASS_TO_PASS via conda testbed pytest, 1800s - config: max_turns 250, env swe_agentic Single-turn swe_bench_verified is untouched. Co-Authored-By: Claude --- evalharness/agent/envs/swe_agentic.py | 219 ++++++++++++++++++ evalharness/config/default.yaml | 6 + .../data/datasets/swe_bench_verified.py | 45 ++-- .../datasets/swe_bench_verified_agentic.py | 26 +++ evalharness/eval/recipes/agent.py | 54 +++++ 5 files changed, 327 insertions(+), 23 deletions(-) create mode 100644 evalharness/agent/envs/swe_agentic.py create mode 100644 evalharness/data/datasets/swe_bench_verified_agentic.py diff --git a/evalharness/agent/envs/swe_agentic.py b/evalharness/agent/envs/swe_agentic.py new file mode 100644 index 0000000..7bc968c --- /dev/null +++ b/evalharness/agent/envs/swe_agentic.py @@ -0,0 +1,219 @@ +"""swe_bench_verified_agentic: multi-turn SWE agent (mini-swe-agent protocol). + +Per sample: start a LONG-RUNNING per-instance Docker container (the official +swebench image, /testbed workdir), give the model a single `bash` tool whose +execs run inside it, loop until the sentinel submission or max_turns, then +recover the patch (sentinel payload, else `git diff` in /testbed). + +Ports es's swe_bench_agentic_adapter (which itself mirrors mini-swe-agent's +swebench.yaml). Scoring stays in the recipe (official swebench harness). +""" + +import asyncio +import json +import re +import subprocess +import uuid +from typing import Any, Dict + +from ...data.sample import ChatMessage, Sample +from ..loop import Environment, register_env + +SENTINEL = 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT' + +# mini-swe-agent swebench.yaml contract (verbatim sections that matter) +INSTANCE_TEMPLATE = """ +Consider the following PR description: +{problem_statement} + + + +# Task Instructions + +## Overview + +You're a software engineer interacting continuously with a computer by submitting commands. +You'll be helping implement necessary changes to meet requirements in the PR description. +Your task is specifically to make changes to non-test files in the current directory in order to fix the issue described in the PR description in a way that is general and consistent with the codebase. +This is an interactive process where you will think and issue AT LEAST ONE command, see the result, then think and issue your next command(s). + +For each response: +1. Include a THOUGHT section explaining your reasoning and what you're trying to accomplish +2. Provide one or more bash tool calls to execute + +## Important Boundaries + +- MODIFY: Regular source code files in /testbed (this is the working directory for all your subsequent commands) +- DO NOT MODIFY: Tests, configuration files (pyproject.toml, setup.cfg, etc.) + +## Recommended Workflow + +1. Analyze the codebase by finding and reading relevant files +2. Create a script to reproduce the issue +3. Edit the source code to resolve the issue +4. Verify your fix works by running your script again +5. Test edge cases to ensure your fix is robust + +## Command Execution Rules + +- Directory or environment variable changes are not persistent; every action runs in a new subshell +- Prefix actions with `cd /testbed && ...` when needed +- Always use non-interactive flags (-y, -f); avoid vi/nano + +## Submission + +When you've completed your work, you MUST submit your changes as a git patch: + +Step 1: Create the patch file +Run `git diff -- path/to/file1 path/to/file2 > patch.txt` listing only the source files you modified. +Do NOT commit your changes. + +Step 2: Verify the patch +Inspect patch.txt to confirm it only contains your intended changes and headers show `--- a/` and `+++ b/` paths. + +Step 3: Submit (EXACT command required) +You MUST use this EXACT command to submit: + +```bash +echo {sentinel} && cat patch.txt +``` + +If the command fails (nonzero exit status), it will not submit. + + +- Creating/viewing the patch and submitting MUST be separate commands (not combined with &&). +- You CANNOT continue working after submitting. + +""" + +# OpenAI function-calling bash tool (mini-swe-agent mainline protocol) +BASH_TOOL = { + 'type': 'function', + 'function': { + 'name': 'bash', + 'description': 'Execute a bash command in /testbed (persistent working ' + 'tree, fresh subshell per call). Use for exploring, ' + 'editing, testing.', + 'parameters': {'type': 'object', + 'properties': {'command': {'type': 'string', + 'description': 'the bash command to run'}}, + 'required': ['command']}, + }, +} + + +def _docker(args, timeout=300): + return subprocess.run(['docker'] + args, capture_output=True, text=True, + timeout=timeout) + + +@register_env('swe_agentic') +class SWEAgenticEnvironment(Environment): + """Self-running env: one persistent container per sample.""" + + def __init__(self): + self.container = '' + + # ---- container lifecycle ---- + def _start(self, image: str) -> str: + name = f'eh-swe-{uuid.uuid4().hex[:10]}' + r = _docker(['run', '-d', '--name', name, + '-w', '/testbed', + '-e', 'PAGER=cat', '-e', 'MANPAGER=cat', + '-e', 'LESS=-R', '-e', 'PIP_PROGRESS_BAR=off', + '-e', 'TQDM_DISABLE=1', + image, 'tail', '-f', '/dev/null'], timeout=120) + if r.returncode != 0: + raise RuntimeError(f'start container failed: {r.stderr[:200]}') + return name + + def _exec(self, cmd: str, timeout: int = 600) -> Dict[str, Any]: + # bash -lc: swebench images activate the per-instance testbed via + # shell startup files (es parity: _SWE_BENCH_INTERPRETER) + r = _docker(['exec', self.container, 'bash', '-lc', cmd], timeout=timeout) + out = (r.stdout or '') + (r.stderr or '') + # cap observation: agents choke on 100k-char dumps + if len(out) > 30000: + out = out[:15000] + '\n...[truncated]...\n' + out[-15000:] + return {'exit': r.returncode, 'out': out} + + def _stop(self): + if self.container: + _docker(['rm', '-f', self.container], timeout=60) + self.container = '' + + # ---- the agent loop ---- + async def run_task(self, adapter, sample: Sample, max_turns: int = 250, + system: str = '', user_adapter=None, gen_kwargs=None, + **kw) -> Dict[str, Any]: + image = (sample.sandbox and sample.sandbox.image) or '' + if not image: + raise RuntimeError('swe_agentic: sample has no sandbox image ' + '(dataset plugin must declare it)') + self.container = self._start(image) + ps = (sample.metadata or {}).get('problem_statement') or sample.input_text + messages = [ChatMessage(role='user', content=INSTANCE_TEMPLATE.format( + problem_statement=ps, sentinel=SENTINEL))] + tools = [{'name': BASH_TOOL['function']['name'], + 'description': BASH_TOOL['function']['description'], + 'parameters': BASH_TOOL['function']['parameters']['properties']}] + + from ...model.output import Usage + total_usage = Usage() + patch = '' + turns = 0 + try: + for turns in range(1, max_turns + 1): + out = await adapter.generate(messages, tools=tools, + **(gen_kwargs or {})) + total_usage = total_usage + out.usage + text = out.text or '' + if SENTINEL in text: + # payload after the sentinel echo = `cat patch.txt` output + patch = text.split(SENTINEL, 1)[1].strip() + break + if out.tool_calls: + messages.append(ChatMessage(role='assistant', content=text)) + for tc in out.tool_calls: + cmd = (tc.arguments_dict or {}).get('command', '') + res = await asyncio.to_thread(self._exec, cmd) + messages.append(ChatMessage( + role='user', + content=f"exit={res['exit']}\n{res['out']}")) + if SENTINEL in res['out']: + # sentinel appeared in command output: capture patch + patch = res['out'].split(SENTINEL, 1)[1].strip() + else: + # no tool call: nudge per protocol + messages.append(ChatMessage(role='assistant', content=text)) + messages.append(ChatMessage( + role='user', + content='Continue: issue a bash tool call (or submit ' + f'via `echo {SENTINEL} && cat patch.txt`).')) + if not patch: + # fallback (es parity): recover from the working tree + res = await asyncio.to_thread( + self._exec, 'cd /testbed && git diff') + patch = res['out'].strip() + finally: + self._stop() + + md = sample.metadata or {} + return { + 'raw': patch or '(no patch produced)', + 'usage': total_usage.model_dump(), + 'env_state': { + 'patch': patch, + 'instance_id': md.get('instance_id'), + 'repo': md.get('repo'), + 'base_commit': md.get('base_commit'), + 'test_patch': md.get('test_patch'), + 'FAIL_TO_PASS': md.get('FAIL_TO_PASS'), + 'PASS_TO_PASS': md.get('PASS_TO_PASS'), + 'image': image, + 'turns_used': turns, + }, + 'trajectory': [{'role': m.role, 'content': (m.content or '')[:2000]} + for m in messages], + 'group_key': str(md.get('instance_id') or ''), + } diff --git a/evalharness/config/default.yaml b/evalharness/config/default.yaml index 58d8533..f7a1d04 100644 --- a/evalharness/config/default.yaml +++ b/evalharness/config/default.yaml @@ -66,3 +66,9 @@ tau2_bench: swe_bench_verified: max_tokens: 32768 # patch 生成预算（diff 可能较长） temperature: 0.0 # es 口径：确定性生成 patch + +swe_bench_verified_agentic: + max_tokens: 4096 # 单轮 bash 命令生成预算（mini-swe-agent 口径） + temperature: 0.0 + env: swe_agentic # 多轮 agent：bash 探索 /testbed + sentinel 提交 + max_turns: 250 # mini-swe-agent 默认步数 diff --git a/evalharness/data/datasets/swe_bench_verified.py b/evalharness/data/datasets/swe_bench_verified.py index 13080cb..0a074ef 100644 --- a/evalharness/data/datasets/swe_bench_verified.py +++ b/evalharness/data/datasets/swe_bench_verified.py @@ -17,28 +17,27 @@ from ..spec import DatasetSpec description='SWE-bench Verified; per-instance docker image carried in Sample.sandbox.', ) ) -def swe_bench_verified(): - def to_sample(record: dict) -> Sample: - instance_id = record['instance_id'] - # EXACT Docker Hub naming (verified by es's pull state: 500/500): - # swebench/sweb.eval.x86_64.{instance_id.lower() with __->_1776_}:latest - # -- per-instance images ARE published under the swebench/ namespace - img = ('swebench/sweb.eval.x86_64.' - + instance_id.lower().replace('__', '_1776_') + ':latest') - return Sample( - input=record['problem_statement'], - target=record['patch'], # gold patch (for oracle/oracle-check only) - sandbox=SandboxSpec(image=img), - metadata={ - 'instance_id': instance_id, - 'repo': record['repo'], - 'base_commit': record['base_commit'], - 'test_patch': record['test_patch'], - 'FAIL_TO_PASS': record['FAIL_TO_PASS'], - 'PASS_TO_PASS': record['PASS_TO_PASS'], - 'environment_setup_commit': record.get('environment_setup_commit'), - 'difficulty': record.get('difficulty'), - }, - ) +def _record_to_sample(record: dict) -> Sample: + instance_id = record['instance_id'] + # EXACT Docker Hub naming (verified by es's pull state: 500/500): + # swebench/sweb.eval.x86_64.{instance_id.lower() with __->_1776_}:latest + # -- per-instance images ARE published under the swebench/ namespace + img = ('swebench/sweb.eval.x86_64.' + + instance_id.lower().replace('__', '_1776_') + ':latest') + return Sample( + input=record['problem_statement'], + target=record['patch'], # gold patch (for oracle/oracle-check only) + sandbox=SandboxSpec(image=img), + metadata={ + 'instance_id': instance_id, + 'repo': record['repo'], + 'base_commit': record['base_commit'], + 'test_patch': record['test_patch'], + 'FAIL_TO_PASS': record['FAIL_TO_PASS'], + 'PASS_TO_PASS': record['PASS_TO_PASS'], + 'environment_setup_commit': record.get('environment_setup_commit'), + 'difficulty': record.get('difficulty'), + }, + ) return to_sample diff --git a/evalharness/data/datasets/swe_bench_verified_agentic.py b/evalharness/data/datasets/swe_bench_verified_agentic.py new file mode 100644 index 0000000..ef09260 --- /dev/null +++ b/evalharness/data/datasets/swe_bench_verified_agentic.py @@ -0,0 +1,26 @@ +"""swe_bench_verified_agentic: same data as swe_bench_verified, agentic bench. + +Separate bench name so both variants coexist (single-turn oracle vs +multi-turn agent); source/split/fields identical — the difference lives +in the recipe (env loop + official harness scoring) and config.""" + +from ..registry import register_dataset +from ..spec import DatasetSpec + + +@register_dataset( + DatasetSpec( + name='swe_bench_verified_agentic', + source='princeton-nlp/SWE-bench_Verified', + split='test', + task_type='agent', + tags=['code', 'agent', 'swe'], + requires=['docker'], + description='SWE-bench Verified (agentic): multi-turn bash agent in ' + 'the per-instance /testbed container.', + ) +) +def swe_bench_verified_agentic(): + from .swe_bench_verified import _record_to_sample + + return _record_to_sample diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index 6a90ba3..12c5ec9 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -261,6 +261,60 @@ def swe_bench_verified(): ) +def _swe_agentic_harness(sample, pred: str): + """Agentic scoring: pred is the recovered patch (from env_state); same + in-container test protocol as the single-turn variant.""" + md = sample.metadata or {} + es = md.get('env_state') or {} + patch = es.get('patch') or pred or '' + f2p = _json.loads(es.get('FAIL_TO_PASS') or '[]') if isinstance( + es.get('FAIL_TO_PASS'), str) else (es.get('FAIL_TO_PASS') or []) + p2p = _json.loads(es.get('PASS_TO_PASS') or '[]') if isinstance( + es.get('PASS_TO_PASS'), str) else (es.get('PASS_TO_PASS') or []) + test_patch = es.get('test_patch') or md.get('test_patch') or '' + tests = f2p + p2p[:20] + script = f'''set -e +cd /testbed +cat > /work/model_patch.diff << 'EOFP' +{patch} +EOFP +cat > /work/test_patch.diff << 'EOFT' +{test_patch} +EOFT +git apply --whitespace=fix /work/model_patch.diff || {{ echo PATCH_FAILED; exit 2; }} +git apply --whitespace=fix /work/test_patch.diff || {{ echo TESTPATCH_FAILED; exit 2; }} +FAIL=0 +while IFS= read -r t; do + [ -z "$t" ] && continue + if ! (conda run -n testbed python -m pytest -x -q "$t" > /dev/null 2>&1); then + echo "TEST_FAILED $t"; FAIL=1 + fi +done <<'EOF' +{chr(10).join(tests)} +EOF +[ "$FAIL" = 0 ] && echo RESOLVED +exit $FAIL +''' + return {'run.sh': script} + + +@register_eval('swe_bench_verified_agentic') +def swe_bench_verified_agentic(): + return EvalRecipe( + name='swe_bench_verified_agentic', + extract='identity', + scorers={'resolved': {'name': 'execution', + 'harness': _swe_agentic_harness, + 'entry': 'run.sh', 'sandbox': 'docker', + 'timeout_s': 1800}}, + exec_workers=2, + description='SWE-bench Verified AGENTIC (mini-swe-agent protocol): ' + 'multi-turn bash agent explores /testbed in the ' + 'per-instance container, submits a git diff; scored by ' + 'FAIL_TO_PASS(+P2P) with the official test patch applied.', + ) + + def _tau2_reward(pred, target, sample, ctx): """Score from the official engine's reward_info (env_state).