370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
"""Structured model output -- the hinge between single-turn eval and agents.
|
|
|
|
Every ModelAdapter returns ModelOutput, never a bare string:
|
|
- single-turn recipes read .text
|
|
- agent loops read .tool_calls and feed observations back
|
|
- accounting/monitoring reads .usage
|
|
"""
|
|
|
|
import time
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class ToolCall(BaseModel):
|
|
"""One function call the model wants executed (OpenAI tool_calls shape)."""
|
|
|
|
id: str = ''
|
|
name: str
|
|
arguments: str = '' # JSON-encoded args string
|
|
arguments_dict: Dict[str, Any] = Field(default_factory=dict) # parsed convenience
|
|
|
|
def to_openai(self) -> Dict[str, Any]:
|
|
return {'id': self.id or f'call_{self.name}', 'type': 'function',
|
|
'function': {'name': self.name, 'arguments': self.arguments or '{}'}}
|
|
|
|
|
|
class Usage(BaseModel):
|
|
input_tokens: int = 0
|
|
output_tokens: int = 0
|
|
total_tokens: int = 0
|
|
cost: float = 0.0
|
|
latency_s: float = 0.0
|
|
finish_reason: str = ''
|
|
# --- performance profile (collected per request; None = not measured) ---
|
|
ttft_s: Optional[float] = None # time to FIRST token (streaming only)
|
|
itl_mean_s: Optional[float] = None # mean inter-token latency (streaming)
|
|
retries: int = 0 # retries consumed before success
|
|
http_status: Optional[int] = None # final HTTP status (e.g. 200)
|
|
|
|
def __add__(self, other: 'Usage') -> 'Usage':
|
|
def _sum_opt(a, b):
|
|
vals = [v for v in (a, b) if v is not None]
|
|
return sum(vals) / len(vals) if len(vals) == 2 else (vals[0] if vals else None)
|
|
|
|
return Usage(
|
|
input_tokens=self.input_tokens + other.input_tokens,
|
|
output_tokens=self.output_tokens + other.output_tokens,
|
|
total_tokens=self.total_tokens + other.total_tokens,
|
|
cost=round(self.cost + other.cost, 6),
|
|
latency_s=round(self.latency_s + other.latency_s, 3),
|
|
finish_reason=self.finish_reason or other.finish_reason,
|
|
ttft_s=_sum_opt(self.ttft_s, other.ttft_s),
|
|
itl_mean_s=_sum_opt(self.itl_mean_s, other.itl_mean_s),
|
|
retries=self.retries + other.retries,
|
|
http_status=self.http_status or other.http_status,
|
|
)
|
|
|
|
|
|
class ModelOutput(BaseModel):
|
|
"""What every adapter returns. text may be '' when the model only calls tools."""
|
|
|
|
text: str = ''
|
|
tool_calls: List[ToolCall] = Field(default_factory=list)
|
|
usage: Usage = Field(default_factory=Usage)
|
|
raw: Optional[Dict[str, Any]] = None # provider response (audit/retry)
|
|
model: str = ''
|
|
created_at: str = Field(default_factory=lambda: time.strftime('%Y-%m-%d %H:%M:%S'))
|
|
|
|
@property
|
|
def is_tool_call(self) -> bool:
|
|
return bool(self.tool_calls)
|