370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
31 lines
1.1 KiB
Python
31 lines
1.1 KiB
Python
"""evalharness.agent -- evaluation driver for agent benchmarks.
|
|
|
|
NOT a general agent framework: no thinking policies. A message pump that
|
|
lets the model under test act through Environment plugins and records
|
|
trajectories for env_reward scorers. Environments and scoring backends are
|
|
registered plugins -- drop a module in agent/envs/ to add one.
|
|
"""
|
|
|
|
import importlib
|
|
import pkgutil
|
|
from pathlib import Path
|
|
|
|
from .loop import (ENV_REGISTRY, Environment, Trajectory, drive,
|
|
get_env, register_env, trajectory_to_prediction)
|
|
|
|
|
|
def _discover_builtin_envs() -> None:
|
|
pkg_dir = Path(__file__).parent / 'envs'
|
|
for info in pkgutil.iter_modules([str(pkg_dir)]):
|
|
importlib.import_module(f'{__name__}.envs.{info.name}')
|
|
|
|
|
|
_discover_builtin_envs()
|
|
|
|
from .envs.bfcl_mock import (BACKEND_REGISTRY, BFCLEnvironment, # noqa: E402,F401
|
|
get_backend, register_backend)
|
|
|
|
__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction',
|
|
'BFCLEnvironment', 'ENV_REGISTRY', 'register_env', 'get_env',
|
|
'BACKEND_REGISTRY', 'register_backend', 'get_backend']
|