204 lines
6.8 KiB
Python
204 lines
6.8 KiB
Python
"""End-to-end: opencode run → bridge (OpenAI Responses API) → DashScope qwen-plus.
|
|
|
|
Skipped by default — opt-in with ``EVALSCOPE_REAL_QWEN=1`` plus a valid
|
|
``DASHSCOPE_API_KEY`` so CI never depends on the network.
|
|
|
|
Covers two tiers:
|
|
|
|
* **Tier A (local)**: requires ``opencode`` CLI installed on the host
|
|
(``npm install -g opencode-ai``). Exercises ``LocalAgentEnvironment``
|
|
+ bridge round-trip without Docker overhead.
|
|
* **Tier B (Docker)**: requires Docker daemon + pre-built
|
|
``evalscope-opencode:latest`` image. Exercises the full
|
|
``EnclaveAgentEnvironment`` path — container → bridge →
|
|
DashScope → trajectory capture.
|
|
|
|
OpenCode speaks the OpenAI **Responses API** (``/openai/v1/responses``),
|
|
not Chat Completions. The bridge's Responses route translates the request
|
|
to ``ChatMessage[]`` for the upstream model, so this test proves the
|
|
complete cross-protocol chain:
|
|
``opencode (Responses API) → bridge → DashScope (OpenAI compat) → qwen-plus``.
|
|
"""
|
|
|
|
import os
|
|
import pytest
|
|
import shutil
|
|
|
|
from evalscope.agent.external import ExternalAgentConfig
|
|
from evalscope.agent.external.adapter import run_external_agent
|
|
from evalscope.api.agent import EventType
|
|
from evalscope.api.dataset import Sample
|
|
from evalscope.api.model import GenerateConfig, Model
|
|
from evalscope.models.openai_compatible import OpenAICompatibleAPI
|
|
from evalscope.utils.function_utils import AsyncioLoopRunner
|
|
|
|
|
|
def _load_env_file() -> None:
|
|
try:
|
|
from dotenv import load_dotenv
|
|
except ImportError:
|
|
return
|
|
load_dotenv(override=False)
|
|
|
|
|
|
_load_env_file()
|
|
|
|
DASHSCOPE_BASE_URL = os.environ.get(
|
|
'DASHSCOPE_BASE_URL', 'https://dashscope.aliyuncs.com/compatible-mode/v1'
|
|
)
|
|
DASHSCOPE_API_KEY = os.environ.get('DASHSCOPE_API_KEY', '')
|
|
TARGET_MODEL = os.environ.get('EVALSCOPE_QWEN_MODEL', 'qwen-plus')
|
|
|
|
|
|
def _has_opencode_cli() -> bool:
|
|
return shutil.which('opencode') is not None
|
|
|
|
|
|
def _docker_available() -> bool:
|
|
if shutil.which('docker') is None:
|
|
return False
|
|
import subprocess
|
|
return subprocess.run(['docker', 'info'], capture_output=True).returncode == 0
|
|
|
|
|
|
def _image_exists(name: str) -> bool:
|
|
import subprocess
|
|
result = subprocess.run(
|
|
['docker', 'images', '-q', name],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return bool(result.stdout.strip())
|
|
|
|
|
|
_REQUIRES_REAL = pytest.mark.skipif(
|
|
os.environ.get('EVALSCOPE_REAL_QWEN') != '1' or not DASHSCOPE_API_KEY,
|
|
reason='real-network test; set EVALSCOPE_REAL_QWEN=1 and DASHSCOPE_API_KEY (e.g. in .env)',
|
|
)
|
|
|
|
_REQUIRES_DOCKER = pytest.mark.skipif(
|
|
os.environ.get('EVALSCOPE_DOCKER_E2E') != '1',
|
|
reason='docker e2e test; set EVALSCOPE_DOCKER_E2E=1 to enable',
|
|
)
|
|
|
|
_REQUIRES_DOCKER_DAEMON = pytest.mark.skipif(
|
|
not _docker_available(),
|
|
reason='docker daemon not reachable',
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _release_bridge_loop():
|
|
yield
|
|
AsyncioLoopRunner.shutdown_for_thread()
|
|
|
|
|
|
def _build_qwen_model() -> Model:
|
|
api = OpenAICompatibleAPI(
|
|
model_name=TARGET_MODEL,
|
|
base_url=DASHSCOPE_BASE_URL,
|
|
api_key=DASHSCOPE_API_KEY,
|
|
)
|
|
return Model(api=api, config=GenerateConfig(max_tokens=256))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tier A: local opencode CLI → bridge → DashScope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@_REQUIRES_REAL
|
|
@pytest.mark.skipif(not _has_opencode_cli(), reason='opencode CLI not installed')
|
|
def test_opencode_local_through_bridge_to_qwen():
|
|
"""End-to-end: opencode run → bridge /openai/v1/chat/completions → qwen-plus."""
|
|
model = _build_qwen_model()
|
|
sample = Sample(input='What is 6 * 7? Reply with just the number.', target='42', id=1)
|
|
config = ExternalAgentConfig(
|
|
framework='opencode',
|
|
kwargs={
|
|
'model_name': TARGET_MODEL,
|
|
'auto_install': False,
|
|
'home_override': '',
|
|
},
|
|
environment='local',
|
|
timeout=180.0,
|
|
)
|
|
result = run_external_agent(config=config, model=model, sample=sample)
|
|
text = (result.output.message.text or '').strip()
|
|
assert text, f'empty agent output; trace_events={[e.type for e in result.trace.events]}'
|
|
assert '42' in text, f'unexpected agent output: {text!r}'
|
|
trace = result.trace
|
|
assert trace.framework == 'opencode'
|
|
assert any(ev.type == EventType.MODEL_GENERATE for ev in trace.events)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tier B: Docker (evalscope-opencode:latest) → bridge → DashScope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@_REQUIRES_REAL
|
|
@_REQUIRES_DOCKER
|
|
@_REQUIRES_DOCKER_DAEMON
|
|
@pytest.mark.skipif(
|
|
not _image_exists('evalscope-opencode:latest'),
|
|
reason='evalscope-opencode:latest image not built; run: '
|
|
'docker build -f evalscope/agent/external/dockerfiles/Dockerfile.opencode '
|
|
'-t evalscope-opencode:latest .',
|
|
)
|
|
def test_opencode_docker_through_bridge_to_qwen():
|
|
"""Docker container (pre-built image) → bridge → qwen-plus via DashScope.
|
|
|
|
Validates the full production path: container sandbox → bridge →
|
|
DashScope → trajectory capture, with a real LLM response.
|
|
"""
|
|
from evalscope.agent.environments.enclave import EnclaveAgentEnvironment
|
|
|
|
model = _build_qwen_model()
|
|
sample = Sample(input='What is 6 * 7? Reply with just the number.', target='42', id=1)
|
|
|
|
env = EnclaveAgentEnvironment(
|
|
engine='docker',
|
|
sandbox_config={
|
|
'image': 'evalscope-opencode:latest',
|
|
'working_dir': '/workspace',
|
|
'network_enabled': True,
|
|
},
|
|
timeout=300.0,
|
|
)
|
|
|
|
config = ExternalAgentConfig(
|
|
framework='opencode',
|
|
kwargs={
|
|
'model_name': TARGET_MODEL,
|
|
'auto_install': False,
|
|
'home_override': '',
|
|
},
|
|
environment='enclave',
|
|
timeout=180.0,
|
|
)
|
|
|
|
result = run_external_agent(
|
|
config=config,
|
|
model=model,
|
|
sample=sample,
|
|
environment_override=env,
|
|
)
|
|
|
|
text = (result.output.message.text or '').strip()
|
|
assert text, f'empty agent output; trace_events={[e.type for e in result.trace.events]}'
|
|
assert '42' in text, f'unexpected agent output: {text!r}'
|
|
|
|
# Trace validation
|
|
trace = result.trace
|
|
assert trace.framework == 'opencode'
|
|
assert trace.environment == 'enclave'
|
|
assert any(ev.type == EventType.MODEL_GENERATE for ev in trace.events)
|
|
assert any(ev.type == EventType.RUN_START for ev in trace.events)
|
|
assert any(ev.type == EventType.RUN_END for ev in trace.events)
|
|
|
|
# Messages validation: bridge should have captured the conversation
|
|
assert result.messages, 'no messages captured by bridge'
|
|
roles = [m.role for m in result.messages]
|
|
assert 'assistant' in roles, f'no assistant message in {roles}'
|