"""Agent evaluation driver: a MESSAGE PUMP, not a thinking framework. We are EVALUATING agents, not being one: the model under test does whatever thinking it wants; this loop only (1) hands tool declarations over, (2) executes the model's tool_calls against an Environment plugin, (3) feeds observations back, (4) stops at max-turns / no-more-calls / env-done. Everything is recorded as a trajectory for env_reward scorers. """ import time from typing import Any, Dict, List, Optional from ..data.sample import ChatMessage, Sample from ..model.adapter import ModelAdapter from ..model.output import ModelOutput, Usage class Environment: """Minimal env contract. One instance per sample. Two execution models, auto-selected by the runner: - MESSAGE PUMP (default): implement reset/step/final_state; the runner drives the model turn-by-turn (bfcl_mock style). - SELF-RUNNING (official engines): implement run_task(adapter, sample); the whole simulation happens inside (official tau2/swe bundles), and a prediction dict is returned directly. Falls back to the pump when run_task is not overridden. """ name = 'base' needs_adapter = False # self-running engines set True; the runner passes # the adapter to the constructor (no name hardcoding) def __init__(self, adapter=None): self.adapter = adapter def reset(self, sample: Sample) -> List[ChatMessage]: return [] async def step(self, tool_calls: List[Any], text: str, sample: Sample ) -> List[ChatMessage]: raise NotImplementedError def final_state(self) -> Dict[str, Any]: return {} async def run_task(self, adapter, sample: Sample, **kw) -> Optional[Dict[str, Any]]: """Self-running hook. Return a prediction dict (raw/trajectory/ env_state/usage/group_key) or None to fall back to the pump.""" return None # ---------------- environment registry (万物皆可插件: envs too) ---------------- from ..eval.registry import EvalRegistry # noqa: E402 ENV_REGISTRY = EvalRegistry('environment') def register_env(name: str): """Class decorator: @register_env('bfcl_mock'). One env per benchmark family; drop a module in agent/envs/ and it auto-registers.""" def decorator(cls): ENV_REGISTRY.register(name, cls) return cls return decorator def get_env(name: str, adapter=None): cls = ENV_REGISTRY.get(name) return cls(adapter=adapter) class Trajectory: """Recorded turns: role-tagged messages + per-turn usage.""" def __init__(self): self.messages: List[Dict[str, Any]] = [] self.turns = 0 self.usage = Usage() self.env_state: Dict[str, Any] = {} def add(self, role: str, content: str, **extra) -> None: entry: Dict[str, Any] = {'role': role, 'content': content, 'turn': self.turns} entry.update(extra) self.messages.append(entry) async def drive( adapter: ModelAdapter, sample: Sample, env: Optional[Environment] = None, max_turns: int = 8, system: str = '', ) -> Trajectory: """Run one sample through the model (+env if present). No env -> single-turn fast path (one generate, done). With env-> pump until the model stops calling tools / env says done / max_turns reached. """ traj = Trajectory() base: List[ChatMessage] = [] if system: base.append(ChatMessage(role='system', content=system)) base += ([ChatMessage(role='user', content=sample.input)] if isinstance(sample.input, str) else list(sample.input)) if env is not None: base += env.reset(sample) if getattr(adapter, 'name', '') == 'mock' \ and adapter.extra.get('mode') in ('boxed', 'oracle', 'fc') \ and sample.target not in ('', None): base = base + [ChatMessage(role='user', content=f'MOCKTARGET::{sample.target}')] tools = [{'name': t.name, 'description': t.description or '', 'parameters': t.parameters} for t in (sample.tools or [])] or None messages = list(base) t0 = time.time() for turn in range(max_turns + 1): out: ModelOutput = await adapter.generate(messages, tools=tools) traj.turns = turn + 1 traj.usage = traj.usage + out.usage traj.add('assistant', out.text, tool_calls=[c.model_dump() for c in out.tool_calls]) messages.append(ChatMessage(role='assistant', content=out.text or _calls_text(out.tool_calls))) if not out.tool_calls or env is None: break # plain answer or single-turn: done observations = await env.step(out.tool_calls, out.text, sample) for obs in observations: traj.add(obs.role, obs.content) messages.append(obs) if env is not None: traj.env_state = env.final_state() traj.messages.insert(0, {'role': 'meta', 'content': f'turns={traj.turns} ' f'latency={time.time() - t0:.1f}s'}) return traj def _calls_text(calls) -> str: import json return json.dumps([c.to_openai()['function'] for c in calls], ensure_ascii=False) def trajectory_to_prediction(traj: Trajectory) -> Dict[str, Any]: """Fold a trajectory into the runner's prediction-dict shape.""" last_assistant = next((m for m in reversed(traj.messages) if m['role'] == 'assistant'), {}) return { 'raw': last_assistant.get('content', ''), 'trajectory': traj.messages, 'env_state': traj.env_state or None, 'usage': traj.usage.model_dump(), 'group_key': str(last_assistant.get('turn', '')), }