Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tau2: silence per-message DEBUG floods; is_final_chunk only when done

Browse Source 
loguru restricted to WARNING+ (every orchestrator step dumped full
messages -- thousands of lines per bench). AssistantMessage.is_final_
chunk now False on tool-call turns (official adapter semantics: True
means the agent is FINISHED talking); always-True handed the turn
back to the user prematurely mid-action-sequence.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 09:07:53 +00:00
parent d244d1a435
commit fa60065099
1 changed files with 16 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
evalharness/agent/envs/tau2_official.py
View File

@ -12,6 +12,7 @@ The env consumes a Sample whose metadata carries the official Task json
import asyncio import asyncio
import json import json
import os import os
import sys
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from ...data.sample import ChatMessage, Sample from ...data.sample import ChatMessage, Sample
@ -29,6 +30,18 @@ _TAU2_LOCAL_CANDIDATES = [
] ]
def _silence_tau2_logs():
"""tau2's loguru DEBUG dumps every full message to the terminal --
thousands of lines per run. Keep WARNING+ only."""
try:
from loguru import logger
logger.remove()
logger.add(sys.stderr, level='WARNING')
except Exception:
pass
def _ensure_tau2_engine(): def _ensure_tau2_engine():
"""Auto-install the REAL tau2-bench engine when missing. """Auto-install the REAL tau2-bench engine when missing.
@ -78,6 +91,7 @@ def _patch_tau2_generate(adapter, user_adapter=None, gen_kwargs=None) -> None:
return return
_ensure_tau2_engine() _ensure_tau2_engine()
_silence_tau2_logs()
import tau2.utils.llm_utils as llm_utils import tau2.utils.llm_utils as llm_utils
original = llm_utils.generate original = llm_utils.generate
@ -136,7 +150,8 @@ def _patch_tau2_generate(adapter, user_adapter=None, gen_kwargs=None) -> None:
calls.append(TauToolCall(id=c.id or f'call_{c.name}', name=c.name, calls.append(TauToolCall(id=c.id or f'call_{c.name}', name=c.name,
arguments=args or {})) arguments=args or {}))
return AssistantMessage(role='assistant', content=text, return AssistantMessage(role='assistant', content=text,
tool_calls=calls or None, cost=None, tool_calls=calls or None,
is_final_chunk=not calls, cost=None,
usage=None, raw_data=None) usage=None, raw_data=None)
# patch module attr AND every already-imported reference (official # patch module attr AND every already-imported reference (official