31 lines
1.1 KiB
Python
31 lines
1.1 KiB
Python
"""evalharness.agent -- evaluation driver for agent benchmarks.
|
|
|
|
NOT a general agent framework: no thinking policies. A message pump that
|
|
lets the model under test act through Environment plugins and records
|
|
trajectories for env_reward scorers. Environments and scoring backends are
|
|
registered plugins -- drop a module in agent/envs/ to add one.
|
|
"""
|
|
|
|
import importlib
|
|
import pkgutil
|
|
from pathlib import Path
|
|
|
|
from .loop import (ENV_REGISTRY, Environment, Trajectory, drive,
|
|
get_env, register_env, trajectory_to_prediction)
|
|
|
|
|
|
def _discover_builtin_envs() -> None:
|
|
pkg_dir = Path(__file__).parent / 'envs'
|
|
for info in pkgutil.iter_modules([str(pkg_dir)]):
|
|
importlib.import_module(f'{__name__}.envs.{info.name}')
|
|
|
|
|
|
_discover_builtin_envs()
|
|
|
|
from .envs.bfcl_mock import (BACKEND_REGISTRY, BFCLEnvironment, # noqa: E402,F401
|
|
get_backend, register_backend)
|
|
|
|
__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction',
|
|
'BFCLEnvironment', 'ENV_REGISTRY', 'register_env', 'get_env',
|
|
'BACKEND_REGISTRY', 'register_backend', 'get_backend']
|