From d244d1a4357f29b70430ad17fc7a59188c3a47c3 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 09:03:52 +0000 Subject: [PATCH] tau2 bridge: strip leakage from simulator output The user simulator (GLM via our adapter) inlines its scenario reasoning in content as '...instructions...reply' -- passed through unstripped, the AGENT receives the scenario's secret instructions (task goal, disclosure strategy), inflating rewards. Both channels now trimmed at the last . Co-Authored-By: Claude --- evalharness/agent/envs/tau2_official.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/evalharness/agent/envs/tau2_official.py b/evalharness/agent/envs/tau2_official.py index 2f4413c..015352d 100644 --- a/evalharness/agent/envs/tau2_official.py +++ b/evalharness/agent/envs/tau2_official.py @@ -119,6 +119,14 @@ def _patch_tau2_generate(adapter, user_adapter=None, gen_kwargs=None) -> None: import json as _json + # strip thinking leakage: hybrid backends inline reasoning in + # content as '...scenario reasoning...visible reply' -- the + # USER simulator's leak hands the agent the scenario's secret + # instructions (task goal, what to disclose), inflating rewards + text = out.text or '' + if '' in text: + text = text.rsplit('', 1)[-1].strip() + calls = [] for c in out.tool_calls: try: @@ -127,7 +135,7 @@ def _patch_tau2_generate(adapter, user_adapter=None, gen_kwargs=None) -> None: args = {'raw': c.arguments} calls.append(TauToolCall(id=c.id or f'call_{c.name}', name=c.name, arguments=args or {})) - return AssistantMessage(role='assistant', content=out.text or '', + return AssistantMessage(role='assistant', content=text, tool_calls=calls or None, cost=None, usage=None, raw_data=None)