336 lines
12 KiB
Python
336 lines
12 KiB
Python
"""Unit tests for the unified sandbox service layer.
|
|
|
|
These tests focus on the pure-Python surface (engine resolution, config
|
|
builder, service-level bookkeeping) and do **not** require Docker or
|
|
ms_enclave managers to actually start. Integration coverage lives in
|
|
``tests/agent/test_t2_environment.py`` (gated by Docker availability).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import pytest
|
|
from typing import Any, Dict
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
from evalscope.api.benchmark import BenchmarkMeta
|
|
from evalscope.api.mixin.sandbox_mixin import EnclaveSandboxBackend
|
|
from evalscope.api.sandbox import (
|
|
PoolHandle,
|
|
SandboxEngine,
|
|
SandboxHandle,
|
|
SandboxService,
|
|
build_sandbox_config,
|
|
ensure_docker_image_built,
|
|
merge_sandbox_config_dicts,
|
|
normalize_docker_build_context,
|
|
resolve_engine,
|
|
shutdown_sandbox_service,
|
|
)
|
|
from evalscope.config import SandboxTaskConfig, TaskConfig
|
|
from evalscope.run import shutdown_sandbox_service_if_enabled
|
|
|
|
# ===========================================================================
|
|
# Engine resolution
|
|
# ===========================================================================
|
|
|
|
|
|
class TestResolveEngine:
|
|
|
|
def test_none_defaults_to_docker(self):
|
|
assert resolve_engine(None) is SandboxEngine.DOCKER
|
|
|
|
def test_enum_passthrough(self):
|
|
assert resolve_engine(SandboxEngine.VOLCENGINE) is SandboxEngine.VOLCENGINE
|
|
|
|
@pytest.mark.parametrize(
|
|
'name',
|
|
['docker', 'Docker', ' DOCKER '],
|
|
)
|
|
def test_docker_aliases(self, name):
|
|
assert resolve_engine(name) is SandboxEngine.DOCKER
|
|
|
|
@pytest.mark.parametrize(
|
|
'name',
|
|
['volcengine', 'volcano', 'volc', 'Volc'],
|
|
)
|
|
def test_volcengine_aliases(self, name):
|
|
assert resolve_engine(name) is SandboxEngine.VOLCENGINE
|
|
|
|
def test_unknown_raises(self):
|
|
with pytest.raises(ValueError, match='Unknown sandbox engine'):
|
|
resolve_engine('nonexistent')
|
|
|
|
|
|
# ===========================================================================
|
|
# Config builder
|
|
# ===========================================================================
|
|
|
|
|
|
class TestMergeSandboxConfigDicts:
|
|
|
|
def test_later_wins(self):
|
|
out = merge_sandbox_config_dicts({'a': 1, 'b': 2}, {'b': 3, 'c': 4})
|
|
assert out == {'a': 1, 'b': 3, 'c': 4}
|
|
|
|
def test_none_and_empty_skipped(self):
|
|
assert merge_sandbox_config_dicts(None, {}, {'x': 1}) == {'x': 1}
|
|
|
|
def test_returns_new_dict(self):
|
|
a = {'x': 1}
|
|
out = merge_sandbox_config_dicts(a, {'y': 2})
|
|
assert out is not a
|
|
|
|
|
|
class TestBuildSandboxConfig:
|
|
|
|
def test_docker_builds_typed_config(self):
|
|
pytest.importorskip('ms_enclave')
|
|
from ms_enclave.sandbox.model import DockerSandboxConfig
|
|
cfg = build_sandbox_config(
|
|
SandboxEngine.DOCKER,
|
|
{'image': 'python:3.11-slim', 'working_dir': '/ws'},
|
|
)
|
|
assert isinstance(cfg, DockerSandboxConfig)
|
|
assert cfg.image == 'python:3.11-slim'
|
|
|
|
def test_custom_image_build_uses_adapter_context(self, tmp_path):
|
|
dockerfile = tmp_path / 'Dockerfile.custom'
|
|
dockerfile.write_text('FROM python:3.11-slim\n', encoding='utf-8')
|
|
meta = BenchmarkMeta(
|
|
name='custom_sandbox',
|
|
dataset_id='dummy',
|
|
sandbox_config={'image': 'custom-sandbox:latest'},
|
|
)
|
|
cfg = TaskConfig(sandbox={'enabled': True, 'engine': 'docker'})
|
|
backend = EnclaveSandboxBackend(
|
|
benchmark_meta=meta,
|
|
task_config=cfg,
|
|
use_custom_image=True,
|
|
build_context_provider=lambda: (str(tmp_path), str(dockerfile)),
|
|
)
|
|
|
|
with patch('evalscope.api.mixin.sandbox_mixin.ensure_docker_image_built') as ensure_image, \
|
|
patch('evalscope.api.mixin.sandbox_mixin.build_and_acquire_pool_sync', return_value=MagicMock()):
|
|
backend.start()
|
|
|
|
ensure_image.assert_called_once_with(
|
|
'custom-sandbox:latest',
|
|
path=str(tmp_path.resolve()),
|
|
dockerfile='Dockerfile.custom',
|
|
label='Sandbox image',
|
|
)
|
|
|
|
def test_ensure_docker_image_built_normalizes_context(self, tmp_path):
|
|
dockerfile = tmp_path / 'Dockerfile.custom'
|
|
dockerfile.write_text('FROM python:3.11-slim\n', encoding='utf-8')
|
|
|
|
build_ctx, dockerfile_name = normalize_docker_build_context(str(tmp_path), str(dockerfile))
|
|
assert build_ctx == str(tmp_path.resolve())
|
|
assert dockerfile_name == 'Dockerfile.custom'
|
|
|
|
with patch('evalscope.api.sandbox.config_builder.should_build_docker_image', return_value=True), \
|
|
patch('evalscope.api.sandbox.config_builder.build_docker_image') as build_image:
|
|
built = ensure_docker_image_built(
|
|
'custom-sandbox:latest',
|
|
path=str(tmp_path),
|
|
dockerfile=str(dockerfile),
|
|
label='Sandbox image',
|
|
)
|
|
|
|
assert built is True
|
|
build_image.assert_called_once_with(
|
|
'custom-sandbox:latest',
|
|
path=str(tmp_path.resolve()),
|
|
dockerfile='Dockerfile.custom',
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# SandboxService manager caching
|
|
# ===========================================================================
|
|
|
|
|
|
class TestSandboxServiceCache:
|
|
"""Verify the manager cache keys by ``(engine, frozen(manager_config))``."""
|
|
|
|
def _make_mock_manager(self) -> MagicMock:
|
|
m = MagicMock()
|
|
m.start = AsyncMock()
|
|
m.stop = AsyncMock()
|
|
return m
|
|
|
|
def test_same_config_returns_cached_manager(self):
|
|
service = SandboxService()
|
|
fake = self._make_mock_manager()
|
|
|
|
async def _run():
|
|
with patch.object(service, '_construct_manager', return_value=fake) as ctor:
|
|
a = await service.get_or_create_manager(SandboxEngine.DOCKER, {'base_url': 'x'})
|
|
b = await service.get_or_create_manager(SandboxEngine.DOCKER, {'base_url': 'x'})
|
|
assert a is b
|
|
ctor.assert_called_once()
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_different_config_creates_new_manager(self):
|
|
service = SandboxService()
|
|
fake_1 = self._make_mock_manager()
|
|
fake_2 = self._make_mock_manager()
|
|
|
|
async def _run():
|
|
managers = iter([fake_1, fake_2])
|
|
with patch.object(service, '_construct_manager', side_effect=lambda *a, **kw: next(managers)):
|
|
a = await service.get_or_create_manager(SandboxEngine.DOCKER, {'base_url': 'x'})
|
|
b = await service.get_or_create_manager(SandboxEngine.DOCKER, {'base_url': 'y'})
|
|
assert a is not b
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_shutdown_all_async_stops_and_clears(self):
|
|
service = SandboxService()
|
|
fake = self._make_mock_manager()
|
|
|
|
async def _run():
|
|
with patch.object(service, '_construct_manager', return_value=fake):
|
|
await service.get_or_create_manager(SandboxEngine.DOCKER, {})
|
|
assert service._managers # populated
|
|
await service.shutdown_all_async()
|
|
fake.stop.assert_awaited_once()
|
|
assert service._managers == {}
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_shutdown_sandbox_service_noop_when_uncreated(self):
|
|
from evalscope.api.sandbox import service as service_module
|
|
|
|
old_service = service_module._SERVICE
|
|
try:
|
|
service_module._SERVICE = None
|
|
shutdown_sandbox_service()
|
|
finally:
|
|
service_module._SERVICE = old_service
|
|
|
|
def test_shutdown_sandbox_service_uses_existing_singleton(self):
|
|
from evalscope.api.sandbox import service as service_module
|
|
|
|
fake = MagicMock()
|
|
old_service = service_module._SERVICE
|
|
try:
|
|
service_module._SERVICE = fake
|
|
shutdown_sandbox_service()
|
|
fake.shutdown_all.assert_called_once()
|
|
finally:
|
|
service_module._SERVICE = old_service
|
|
|
|
|
|
class TestSandboxServiceRunTeardown:
|
|
|
|
def test_shutdown_skipped_when_sandbox_disabled(self):
|
|
cfg = TaskConfig()
|
|
with patch('evalscope.api.sandbox.shutdown_sandbox_service') as shutdown:
|
|
shutdown_sandbox_service_if_enabled(cfg)
|
|
shutdown.assert_not_called()
|
|
|
|
def test_shutdown_called_when_sandbox_enabled(self):
|
|
cfg = TaskConfig(sandbox={'enabled': True, 'engine': 'docker'})
|
|
with patch('evalscope.api.sandbox.shutdown_sandbox_service') as shutdown:
|
|
shutdown_sandbox_service_if_enabled(cfg)
|
|
shutdown.assert_called_once_with()
|
|
|
|
|
|
# ===========================================================================
|
|
# Handles
|
|
# ===========================================================================
|
|
|
|
|
|
class TestHandles:
|
|
|
|
def test_pool_handle_execute_delegates(self):
|
|
manager = MagicMock()
|
|
manager.execute_tool_in_pool = AsyncMock(return_value='OK')
|
|
handle = PoolHandle(manager)
|
|
|
|
out = asyncio.run(handle.execute_tool('shell_executor', {'command': 'ls'}))
|
|
assert out == 'OK'
|
|
manager.execute_tool_in_pool.assert_awaited_once_with(
|
|
'shell_executor', {'command': 'ls'}
|
|
)
|
|
|
|
def test_sandbox_handle_close_idempotent(self):
|
|
manager = MagicMock()
|
|
manager.delete_sandbox = AsyncMock()
|
|
handle = SandboxHandle(manager, 'sb-1')
|
|
|
|
async def _run():
|
|
await handle.close()
|
|
await handle.close() # second call is no-op
|
|
|
|
asyncio.run(_run())
|
|
manager.delete_sandbox.assert_awaited_once_with('sb-1')
|
|
|
|
def test_sandbox_handle_execute_after_close_raises(self):
|
|
manager = MagicMock()
|
|
manager.delete_sandbox = AsyncMock()
|
|
handle = SandboxHandle(manager, 'sb-1')
|
|
|
|
async def _run():
|
|
await handle.close()
|
|
await handle.execute_tool('x', {})
|
|
|
|
with pytest.raises(RuntimeError, match='already closed'):
|
|
asyncio.run(_run())
|
|
|
|
|
|
# ===========================================================================
|
|
# TaskConfig.sandbox ← legacy fields (legacy → nested one-way fold)
|
|
# ===========================================================================
|
|
|
|
|
|
class TestTaskConfigSandboxBridge:
|
|
|
|
def test_legacy_only_projects_to_nested(self):
|
|
cfg = TaskConfig(
|
|
use_sandbox=True,
|
|
sandbox_type='volcengine',
|
|
sandbox_manager_config={'key': 'val'},
|
|
)
|
|
assert isinstance(cfg.sandbox, SandboxTaskConfig)
|
|
assert cfg.sandbox.enabled is True
|
|
assert cfg.sandbox.engine == 'volcengine'
|
|
assert cfg.sandbox.manager_config == {'key': 'val'}
|
|
|
|
def test_nested_does_not_mirror_back_to_legacy(self):
|
|
# After the mirror was removed, legacy fields keep whatever the user
|
|
# passed in (defaults here) and are NOT synchronised with `sandbox`.
|
|
cfg = TaskConfig(sandbox={'enabled': True, 'engine': 'volc', 'manager_config': {'k': 'v'}})
|
|
assert cfg.sandbox.enabled is True
|
|
assert cfg.sandbox.engine == 'volc'
|
|
assert cfg.sandbox.manager_config == {'k': 'v'}
|
|
# Legacy fields stay at their defaults (not mirrored).
|
|
assert cfg.use_sandbox is False
|
|
assert cfg.sandbox_type == 'docker'
|
|
assert cfg.sandbox_manager_config == {}
|
|
|
|
def test_both_set_nested_wins(self):
|
|
# Nested `sandbox` wins; legacy fields remain untouched inputs.
|
|
cfg = TaskConfig(
|
|
use_sandbox=False, # contradicts the nested value below
|
|
sandbox_type='docker',
|
|
sandbox={'enabled': True, 'engine': 'volcengine', 'manager_config': {'k': 'v'}},
|
|
)
|
|
assert cfg.sandbox.enabled is True
|
|
assert cfg.sandbox.engine == 'volcengine'
|
|
assert cfg.sandbox.manager_config == {'k': 'v'}
|
|
# Legacy fields are NOT mirrored from the nested object.
|
|
assert cfg.use_sandbox is False
|
|
assert cfg.sandbox_type == 'docker'
|
|
|
|
def test_default_config_is_carried_through(self):
|
|
cfg = TaskConfig(sandbox={
|
|
'enabled': True,
|
|
'engine': 'docker',
|
|
'default_config': {'image': 'my:img'},
|
|
})
|
|
assert cfg.sandbox.default_config == {'image': 'my:img'}
|