"""Agent evaluation driver: a MESSAGE PUMP, not a thinking framework. We are EVALUATING agents, not being one: the model under test does whatever thinking it wants; this loop only (1) hands tool declarations over, (2) executes the model's tool_calls against an Environment plugin, (3) feeds observations back, (4) stops at max-turns / no-more-calls / env-done. Everything is recorded as a trajectory for env_reward scorers. """ import time from typing import Any, Dict, List, Optional from ..data.sample import ChatMessage, Sample from ..model.adapter import ModelAdapter from ..model.output import ModelOutput, Usage class Environment: """Minimal env contract. One instance per sample.""" name = 'base' def reset(self, sample: Sample) -> List[ChatMessage]: """Prepare per-sample state; return any extra opening messages (e.g. tool/user-simulator turns). Default: nothing.""" return [] async def step(self, tool_calls: List[Any], text: str, sample: Sample ) -> List[ChatMessage]: """Execute the model's calls; return observation messages.""" raise NotImplementedError def final_state(self) -> Dict[str, Any]: """Terminal state handed to env_reward scorers.""" return {} # ---------------- environment registry (万物皆可插件: envs too) ---------------- from ..eval.registry import EvalRegistry # noqa: E402 ENV_REGISTRY = EvalRegistry('environment') def register_env(name: str): """Class decorator: @register_env('bfcl_mock'). One env per benchmark family; drop a module in agent/envs/ and it auto-registers.""" def decorator(cls): ENV_REGISTRY.register(name, cls) return cls return decorator def get_env(name: str): cls = ENV_REGISTRY.get(name) return cls() class Trajectory: """Recorded turns: role-tagged messages + per-turn usage.""" def __init__(self): self.messages: List[Dict[str, Any]] = [] self.turns = 0 self.usage = Usage() self.env_state: Dict[str, Any] = {} def add(self, role: str, content: str, **extra) -> None: entry: Dict[str, Any] = {'role': role, 'content': content, 'turn': self.turns} entry.update(extra) self.messages.append(entry) async def drive( adapter: ModelAdapter, sample: Sample, env: Optional[Environment] = None, max_turns: int = 8, system: str = '', ) -> Trajectory: """Run one sample through the model (+env if present). No env -> single-turn fast path (one generate, done). With env-> pump until the model stops calling tools / env says done / max_turns reached. """ traj = Trajectory() base: List[ChatMessage] = [] if system: base.append(ChatMessage(role='system', content=system)) base += ([ChatMessage(role='user', content=sample.input)] if isinstance(sample.input, str) else list(sample.input)) if env is not None: base += env.reset(sample) if getattr(adapter, 'name', '') == 'mock' \ and adapter.extra.get('mode') in ('boxed', 'oracle', 'fc') \ and sample.target not in ('', None): base = base + [ChatMessage(role='user', content=f'MOCKTARGET::{sample.target}')] tools = [{'name': t.name, 'description': t.description or '', 'parameters': t.parameters} for t in (sample.tools or [])] or None messages = list(base) t0 = time.time() for turn in range(max_turns + 1): out: ModelOutput = await adapter.generate(messages, tools=tools) traj.turns = turn + 1 traj.usage = traj.usage + out.usage traj.add('assistant', out.text, tool_calls=[c.model_dump() for c in out.tool_calls]) messages.append(ChatMessage(role='assistant', content=out.text or _calls_text(out.tool_calls))) if not out.tool_calls or env is None: break # plain answer or single-turn: done observations = await env.step(out.tool_calls, out.text, sample) for obs in observations: traj.add(obs.role, obs.content) messages.append(obs) if env is not None: traj.env_state = env.final_state() traj.messages.insert(0, {'role': 'meta', 'content': f'turns={traj.turns} ' f'latency={time.time() - t0:.1f}s'}) return traj def _calls_text(calls) -> str: import json return json.dumps([c.to_openai()['function'] for c in calls], ensure_ascii=False) def trajectory_to_prediction(traj: Trajectory) -> Dict[str, Any]: """Fold a trajectory into the runner's prediction-dict shape.""" last_assistant = next((m for m in reversed(traj.messages) if m['role'] == 'assistant'), {}) return { 'raw': last_assistant.get('content', ''), 'trajectory': traj.messages, 'env_state': traj.env_state or None, 'usage': traj.usage.model_dump(), 'group_key': str(last_assistant.get('turn', '')), }