2c3672f2cb
- AdaptiveGate rewritten (Netflix Gradient2): window-vs-window per-stream speed gradient, count-driven windows with admission stamps, no thresholds or mode state machine; failures x0.7 + 30s drain pause - session-level admission for multi-turn agents (_SessionGate): in-progress sessions hold slots until done, newcomers queue at the door; capacity follows the model gate's discovered limit (CONCUR-style continuity) - image service: memory-first register (zero docker calls for known images), TTL-cached docker images listing, optimistic ready when the daemon is unreachable (docker save contention no longer kills runs); es tar loading removed in favor of ModelScope shipping (ms_images.py per-image tar upload/pull with round-trip verification) - runner: circuit breaker (12 consecutive failures abort the bench), first-failure error printed immediately - swe_agentic: image wait / docker run / rm off the event loop; exec timeout becomes an observation the agent can react to; container gets curlrc + git low-speed aborts (stalled github downloads fail fast) - eval run excludes its own endpoints from http_proxy (a sick personal proxy read as 'endpoint dead' and killed whole runs) - progress bar shows failed count; swe agentic exec_workers 2 -> 4 Co-Authored-By: Claude <noreply@anthropic.com>
366 lines
16 KiB
Python
366 lines
16 KiB
Python
"""swe_bench_verified_agentic: multi-turn SWE agent (mini-swe-agent protocol).
|
|
|
|
Per sample: start a LONG-RUNNING per-instance Docker container (the official
|
|
swebench image, /testbed workdir), give the model a single `bash` tool whose
|
|
execs run inside it, loop until the sentinel submission or max_turns, then
|
|
recover the patch (sentinel payload, else `git diff` in /testbed).
|
|
|
|
Ports es's swe_bench_agentic_adapter (which itself mirrors mini-swe-agent's
|
|
swebench.yaml). Scoring stays in the recipe (official swebench harness).
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
from typing import Any, Dict
|
|
|
|
from ...data.sample import ChatMessage, Sample
|
|
from ..loop import Environment, register_env
|
|
|
|
|
|
|
|
SENTINEL = 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT'
|
|
|
|
|
|
class _SessionGate:
|
|
"""Session-level admission for multi-turn agent benches (CONCUR-style).
|
|
|
|
Turn-level fairness lets EVERY started session crawl at 1/N bandwidth
|
|
-- fine for single-turn benches, terrible for 100-turn agents: first
|
|
completions arrive absurdly late and all contexts stay hot. Here a
|
|
session HOLDS its slot for its whole life (execution continuity: an
|
|
in-progress session always outranks a new one), and the number of
|
|
admitted sessions follows the model gate's discovered limit.
|
|
|
|
No preemption: a shrink only blocks NEW sessions; running ones drain
|
|
naturally.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.active = 0
|
|
self._cond = None
|
|
self._loop = None
|
|
|
|
def _limit(self, adapter) -> int:
|
|
gates = getattr(adapter, '_gates', None)
|
|
if gates:
|
|
try:
|
|
return max(1, int(gates[0].limit))
|
|
except Exception:
|
|
pass
|
|
return int(__import__('os').environ.get(
|
|
'EVALHARNESS_SESSION_SLOTS', '8') or 8)
|
|
|
|
async def enter(self, adapter):
|
|
import asyncio
|
|
|
|
loop = asyncio.get_running_loop()
|
|
if self._cond is None or self._loop is not loop:
|
|
self._loop = loop
|
|
self._cond = asyncio.Condition()
|
|
self.active = 0 # fresh loop: nothing in flight
|
|
while self.active >= self._limit(adapter):
|
|
await self._cond.acquire()
|
|
try:
|
|
await self._cond.wait()
|
|
finally:
|
|
self._cond.release()
|
|
self.active += 1
|
|
|
|
async def exit(self):
|
|
import asyncio
|
|
|
|
self.active = max(0, self.active - 1)
|
|
if self._cond is not None:
|
|
try:
|
|
async with self._cond:
|
|
self._cond.notify_all()
|
|
except RuntimeError:
|
|
pass
|
|
|
|
|
|
_SESSION_GATE = _SessionGate()
|
|
|
|
# mini-swe-agent swebench.yaml contract (verbatim sections that matter)
|
|
INSTANCE_TEMPLATE = """<pr_description>
|
|
Consider the following PR description:
|
|
{problem_statement}
|
|
</pr_description>
|
|
|
|
<instructions>
|
|
# Task Instructions
|
|
|
|
## Overview
|
|
|
|
You're a software engineer interacting continuously with a computer by submitting commands.
|
|
You'll be helping implement necessary changes to meet requirements in the PR description.
|
|
Your task is specifically to make changes to non-test files in the current directory in order to fix the issue described in the PR description in a way that is general and consistent with the codebase.
|
|
<IMPORTANT>This is an interactive process where you will think and issue AT LEAST ONE command, see the result, then think and issue your next command(s).</important>
|
|
|
|
For each response:
|
|
1. Include a THOUGHT section explaining your reasoning and what you're trying to accomplish
|
|
2. Provide one or more bash tool calls to execute
|
|
|
|
## Important Boundaries
|
|
|
|
- MODIFY: Regular source code files in /testbed (this is the working directory for all your subsequent commands)
|
|
- DO NOT MODIFY: Tests, configuration files (pyproject.toml, setup.cfg, etc.)
|
|
|
|
## Recommended Workflow
|
|
|
|
1. Analyze the codebase by finding and reading relevant files
|
|
2. Create a script to reproduce the issue
|
|
3. Edit the source code to resolve the issue
|
|
4. Verify your fix works by running your script again
|
|
5. Test edge cases to ensure your fix is robust
|
|
|
|
## Command Execution Rules
|
|
|
|
- Directory or environment variable changes are not persistent; every action runs in a new subshell
|
|
- Prefix actions with `cd /testbed && ...` when needed
|
|
- Always use non-interactive flags (-y, -f); avoid vi/nano
|
|
|
|
## Submission
|
|
|
|
When you've completed your work, you MUST submit your changes as a git patch:
|
|
|
|
Step 1: Create the patch file
|
|
Run `git diff -- path/to/file1 path/to/file2 > patch.txt` listing only the source files you modified.
|
|
Do NOT commit your changes.
|
|
|
|
Step 2: Verify the patch
|
|
Inspect patch.txt to confirm it only contains your intended changes and headers show `--- a/` and `+++ b/` paths.
|
|
|
|
Step 3: Submit (EXACT command required)
|
|
You MUST use this EXACT command to submit:
|
|
|
|
```bash
|
|
echo {sentinel} && cat patch.txt
|
|
```
|
|
|
|
If the command fails (nonzero exit status), it will not submit.
|
|
|
|
<CRITICAL>
|
|
- Creating/viewing the patch and submitting MUST be separate commands (not combined with &&).
|
|
- You CANNOT continue working after submitting.
|
|
</CRITICAL>
|
|
</instructions>"""
|
|
|
|
# OpenAI function-calling bash tool (mini-swe-agent mainline protocol)
|
|
BASH_TOOL = {
|
|
'type': 'function',
|
|
'function': {
|
|
'name': 'bash',
|
|
'description': 'Execute a bash command in /testbed (persistent working '
|
|
'tree, fresh subshell per call). Use for exploring, '
|
|
'editing, testing.',
|
|
'parameters': {'type': 'object',
|
|
'properties': {'command': {'type': 'string',
|
|
'description': 'the bash command to run'}},
|
|
'required': ['command']},
|
|
},
|
|
}
|
|
|
|
|
|
def _docker(args, timeout=300):
|
|
return subprocess.run(['docker'] + args, capture_output=True, text=True,
|
|
timeout=timeout)
|
|
|
|
|
|
@register_env('swe_agentic')
|
|
class SWEAgenticEnvironment(Environment):
|
|
"""Self-running env: one persistent container per sample."""
|
|
|
|
def __init__(self, adapter=None, **_):
|
|
# adapter arg: get_env uniform signature (unused here; the adapter
|
|
# is passed to run_task per sample)
|
|
self.container = ''
|
|
|
|
# ---- container lifecycle ----
|
|
def _start(self, image: str) -> str:
|
|
# per-instance image: ensure (local check -> mirror-chain pull with
|
|
# progress) BEFORE docker run -- `docker run` auto-pulls with zero
|
|
# output and its 120s timeout kills runs on slow mirrors
|
|
# unified image service: pulls happen in the BACKGROUND from task
|
|
# arrival; a not-yet-ready image BLOCKS here instead of failing
|
|
# (the old inline pull timed-out docker run at 120s and killed runs)
|
|
from ...sandbox.image_service import get_image_service
|
|
|
|
if not get_image_service().wait_ready(image, timeout_s=1800):
|
|
raise RuntimeError(f'image {image} unavailable after all sources')
|
|
name = f'eh-swe-{uuid.uuid4().hex[:10]}'
|
|
r = _docker(['run', '-d', '--name', name,
|
|
'-w', '/testbed',
|
|
'-e', 'PAGER=cat', '-e', 'MANPAGER=cat',
|
|
# CN pip mirror: pypi.org is unreachable from this
|
|
# host's containers (agent `pip install` hit 600s
|
|
# timeouts); tuna downloads the same package in ~7s
|
|
'-e', 'PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple',
|
|
'-e', 'LESS=-R', '-e', 'PIP_PROGRESS_BAR=off',
|
|
'-e', 'TQDM_DISABLE=1',
|
|
image, 'tail', '-f', '/dev/null'], timeout=120)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f'start container failed: {r.stderr[:200]}')
|
|
# network hardening: github.com is reachable from these containers
|
|
# but large transfers stall mid-stream (GFW throttling) -- an agent
|
|
# curl/git then hangs until the 600s exec cap. Make curl and git
|
|
# abort stalled transfers themselves so the agent sees a FAST
|
|
# failure and reroutes instead of burning 10 minutes:
|
|
# curl: ~/.curlrc applies to every agent-invoked curl
|
|
# git: lowSpeed abort on <1KB/s sustained 30s
|
|
_docker(['exec', name, 'bash', '-lc',
|
|
'printf "connect-timeout 15\\nspeed-time 30\\nspeed-limit 1024\\n" > /root/.curlrc 2>/dev/null; '
|
|
'git config --global http.lowSpeedLimit 1024 2>/dev/null; '
|
|
'git config --global http.lowSpeedTime 30 2>/dev/null'], timeout=30)
|
|
return name
|
|
|
|
def _exec(self, cmd: str, timeout: int = 600) -> Dict[str, Any]:
|
|
# bash -lc: swebench images activate the per-instance testbed via
|
|
# shell startup files (es parity: _SWE_BENCH_INTERPRETER)
|
|
try:
|
|
r = _docker(['exec', self.container, 'bash', '-lc', cmd],
|
|
timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
# a hung command (e.g. curl to an unreachable github from a CN
|
|
# container) must be an OBSERVATION the agent can react to --
|
|
# letting it raise killed the whole task 600s in, mid-loop,
|
|
# after all its previous turns were already spent
|
|
return {'exit': 124,
|
|
'out': f'COMMAND TIMED OUT after {timeout}s (killed). '
|
|
'The command may be blocked on the network -- '
|
|
'github.com is often unreachable from this '
|
|
'container; try a mirror (ghproxy) or proceed '
|
|
'without the download.'}
|
|
out = (r.stdout or '') + (r.stderr or '')
|
|
# cap observation: agents choke on 100k-char dumps
|
|
if len(out) > 30000:
|
|
out = out[:15000] + '\n...[truncated]...\n' + out[-15000:]
|
|
return {'exit': r.returncode, 'out': out}
|
|
|
|
def _stop(self):
|
|
if self.container:
|
|
_docker(['rm', '-f', self.container], timeout=60)
|
|
self.container = ''
|
|
|
|
# ---- the agent loop ----
|
|
async def run_task(self, adapter, sample: Sample, max_turns: int = 250,
|
|
system: str = '', user_adapter=None, gen_kwargs=None,
|
|
**kw) -> Dict[str, Any]:
|
|
# SESSION admission BEFORE the container: hold arriving sessions at
|
|
# the door instead of letting all of them start containers and then
|
|
# crawl at 1/N turn bandwidth
|
|
await _SESSION_GATE.enter(adapter)
|
|
try:
|
|
return await self._run_session(adapter, sample, max_turns,
|
|
system, user_adapter, gen_kwargs)
|
|
finally:
|
|
await _SESSION_GATE.exit()
|
|
|
|
async def _run_session(self, adapter, sample: Sample, max_turns: int = 250,
|
|
system: str = '', user_adapter=None,
|
|
gen_kwargs=None, **kw) -> Dict[str, Any]:
|
|
image = (sample.sandbox and sample.sandbox.image) or ''
|
|
if not image:
|
|
raise RuntimeError('swe_agentic: sample has no sandbox image '
|
|
'(dataset plugin must declare it)')
|
|
# register on ARRIVAL: the background pool starts pulling while
|
|
# earlier samples are still generating. OFF the loop: register runs
|
|
# `docker images` (2-4s against a 534-image daemon) and 100 tasks
|
|
# doing that serially ON the loop froze the whole runner at startup
|
|
from ...sandbox.image_service import get_image_service
|
|
|
|
svc = get_image_service()
|
|
await asyncio.to_thread(svc.register, [image])
|
|
# NEVER wait on the image barrier on the event loop: one blocked
|
|
# wait_ready froze the WHOLE runner (0.00/s, every other task
|
|
# stalled behind it). Park the wait (and the docker run) in a
|
|
# thread; _start's own wait then returns instantly (already ready)
|
|
if not await asyncio.to_thread(svc.wait_ready, image, 3600):
|
|
raise RuntimeError(f'image {image} unavailable after all sources')
|
|
self.container = await asyncio.to_thread(self._start, image)
|
|
ps = (sample.metadata or {}).get('problem_statement') or sample.input_text
|
|
messages = [ChatMessage(role='user', content=INSTANCE_TEMPLATE.format(
|
|
problem_statement=ps, sentinel=SENTINEL))]
|
|
tools = [{'name': BASH_TOOL['function']['name'],
|
|
'description': BASH_TOOL['function']['description'],
|
|
'parameters': BASH_TOOL['function']['parameters']['properties']}]
|
|
|
|
from ...model.output import Usage
|
|
total_usage = Usage()
|
|
patch = ''
|
|
turns = 0
|
|
try:
|
|
for turns in range(1, max_turns + 1):
|
|
out = await adapter.generate(messages, tools=tools,
|
|
**(gen_kwargs or {}))
|
|
total_usage = total_usage + out.usage
|
|
text = out.text or ''
|
|
if SENTINEL in text:
|
|
# the model MENTIONED the sentinel in prose OR ran it.
|
|
# Never split on the mention -- the model discusses the
|
|
# command and that chatter became the "patch" (145B of
|
|
# conversational English). Read the ACTUAL file the agent
|
|
# created in the container.
|
|
r3 = await asyncio.to_thread(
|
|
self._exec, 'cat /testbed/patch.txt')
|
|
if r3['exit'] == 0 and r3['out'].strip():
|
|
patch = r3['out'].strip() + '\n'
|
|
break
|
|
# patch.txt empty/missing: keep looping, git-diff at end
|
|
if out.tool_calls:
|
|
messages.append(ChatMessage(role='assistant', content=text))
|
|
for tc in out.tool_calls:
|
|
cmd = (tc.arguments_dict or {}).get('command', '')
|
|
res = await asyncio.to_thread(self._exec, cmd)
|
|
messages.append(ChatMessage(
|
|
role='user',
|
|
content=f"exit={res['exit']}\n{res['out']}"))
|
|
if SENTINEL in res['out']:
|
|
# sentinel in command output: the `cat patch.txt`
|
|
# already ran -- read the actual file
|
|
r2 = await asyncio.to_thread(
|
|
self._exec, 'cat /testbed/patch.txt')
|
|
if r2['exit'] == 0 and r2['out'].strip():
|
|
patch = r2['out'].strip() + '\n'
|
|
else:
|
|
# no tool call: nudge per protocol
|
|
messages.append(ChatMessage(role='assistant', content=text))
|
|
messages.append(ChatMessage(
|
|
role='user',
|
|
content='Continue: issue a bash tool call (or submit '
|
|
f'via `echo {SENTINEL} && cat patch.txt`).'))
|
|
if not patch:
|
|
# fallback (es parity): recover from the working tree
|
|
res = await asyncio.to_thread(
|
|
self._exec, 'cd /testbed && git diff')
|
|
patch = res['out'].strip()
|
|
finally:
|
|
await asyncio.to_thread(self._stop)
|
|
|
|
md = sample.metadata or {}
|
|
return {
|
|
'raw': patch or '(no patch produced)',
|
|
'usage': total_usage.model_dump(),
|
|
'env_state': {
|
|
'patch': patch,
|
|
'instance_id': md.get('instance_id'),
|
|
'repo': md.get('repo'),
|
|
'base_commit': md.get('base_commit'),
|
|
'test_patch': md.get('test_patch'),
|
|
'FAIL_TO_PASS': md.get('FAIL_TO_PASS'),
|
|
'PASS_TO_PASS': md.get('PASS_TO_PASS'),
|
|
'version': md.get('version') or '',
|
|
'hints_text': md.get('hints_text') or '',
|
|
'created_at': md.get('created_at') or '',
|
|
'environment_setup_commit': md.get('environment_setup_commit') or '',
|
|
'image': image,
|
|
'turns_used': turns,
|
|
},
|
|
'trajectory': [{'role': m.role, 'content': (m.content or '')[:2000]}
|
|
for m in messages],
|
|
'group_key': str(md.get('instance_id') or ''),
|
|
}
|