Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

swe_agentic: sentinel patch from container file, not text split

Browse Source 
The model DISCUSSES the sentinel command in prose; splitting on the
first occurrence captured conversational English as the 'patch'
(145B of chatter instead of the diff). On sentinel detection we now
read /testbed/patch.txt directly from the container -- the file the
agent actually created. Empty/missing file falls through to git diff.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-18 10:35:39 +00:00
parent fc8ade3c03
commit c7811fa774
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
evalharness/agent/envs/swe_agentic.py
View File

@ -189,11 +189,17 @@ class SWEAgenticEnvironment(Environment):
total_usage = total_usage + out.usage total_usage = total_usage + out.usage
text = out.text or '' text = out.text or ''
if SENTINEL in text: if SENTINEL in text:
# payload after the sentinel echo = `cat patch.txt` output. # the model MENTIONED the sentinel in prose OR ran it.
# A unified diff must end with a newline; a stripped # Never split on the mention -- the model discusses the
# payload fails git apply with 'corrupt patch at line N' # command and that chatter became the "patch" (145B of
patch = text.split(SENTINEL, 1)[1].strip() + '\n' # conversational English). Read the ACTUAL file the agent
break # created in the container.
r3 = await asyncio.to_thread(
self._exec, 'cat /testbed/patch.txt')
if r3['exit'] == 0 and r3['out'].strip():
patch = r3['out'].strip() + '\n'
break
# patch.txt empty/missing: keep looping, git-diff at end
if out.tool_calls: if out.tool_calls:
messages.append(ChatMessage(role='assistant', content=text)) messages.append(ChatMessage(role='assistant', content=text))
for tc in out.tool_calls: for tc in out.tool_calls:
@ -203,8 +209,12 @@ class SWEAgenticEnvironment(Environment):
role='user', role='user',
content=f"exit={res['exit']}\n{res['out']}")) content=f"exit={res['exit']}\n{res['out']}"))
if SENTINEL in res['out']: if SENTINEL in res['out']:
# sentinel appeared in command output: capture patch # sentinel in command output: the `cat patch.txt`
patch = res['out'].split(SENTINEL, 1)[1].strip() + '\n' # already ran -- read the actual file
r2 = await asyncio.to_thread(
self._exec, 'cat /testbed/patch.txt')
if r2['exit'] == 0 and r2['out'].strip():
patch = r2['out'].strip() + '\n'
else: else:
# no tool call: nudge per protocol # no tool call: nudge per protocol
messages.append(ChatMessage(role='assistant', content=text)) messages.append(ChatMessage(role='assistant', content=text))