153 lines
5.4 KiB
Python
153 lines
5.4 KiB
Python
"""Unit tests for the MCP integration.
|
|
|
|
Spawns a tiny FastMCP echo server as a stdio child process and verifies
|
|
that :class:`MCPServer` + :func:`mcp_tools` correctly:
|
|
|
|
1. Initialise the server, list tools, surface them as :class:`ToolInfo`.
|
|
2. Convert the MCP input schema into evalscope's ``ToolParams`` properties.
|
|
3. Invoke a tool and return its text content as a ``ToolHandler``-shaped
|
|
coroutine.
|
|
|
|
A second test exercises the streamable-HTTP transport against the same
|
|
echo server bound to ``127.0.0.1`` — guards the ``client.py`` HTTP path
|
|
that the stdio test does not touch.
|
|
"""
|
|
|
|
import asyncio
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import unittest
|
|
|
|
try:
|
|
import mcp # noqa: F401
|
|
except ImportError:
|
|
raise unittest.SkipTest(
|
|
'mcp extra not installed; install with `pip install evalscope[mcp]` to run these tests.'
|
|
)
|
|
|
|
import evalscope # noqa: F401 - trigger registration
|
|
from evalscope.api.agent.mcp import MCPServer, MCPServerConfigHTTP, MCPServerConfigStdio, mcp_tools
|
|
from evalscope.api.tool import ToolCall
|
|
from evalscope.api.tool.tool_call import ToolFunction
|
|
|
|
|
|
def _echo_server_config() -> MCPServerConfigStdio:
|
|
"""Stdio config that spawns the local echo server fixture."""
|
|
return MCPServerConfigStdio(
|
|
command=sys.executable,
|
|
args=['-m', 'tests.agent.mcp_echo_server'],
|
|
)
|
|
|
|
|
|
def _free_port() -> int:
|
|
"""Reserve a free localhost port for the HTTP fixture."""
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
s.bind(('127.0.0.1', 0))
|
|
return s.getsockname()[1]
|
|
|
|
|
|
def _wait_for_port(port: int, timeout: float = 10.0) -> None:
|
|
"""Block until ``127.0.0.1:port`` accepts connections, or raise."""
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
try:
|
|
s.settimeout(0.5)
|
|
s.connect(('127.0.0.1', port))
|
|
return
|
|
except OSError:
|
|
time.sleep(0.1)
|
|
raise RuntimeError(f'echo HTTP server did not start on port {port} within {timeout}s')
|
|
|
|
|
|
class TestMCPServer(unittest.TestCase):
|
|
"""End-to-end MCP server lifecycle."""
|
|
|
|
def test_list_and_call_echo_tool(self):
|
|
async def _run():
|
|
async with MCPServer(_echo_server_config()) as server:
|
|
handlers, infos = await mcp_tools(server)
|
|
# 1. tools/list — server advertises a single 'echo' tool
|
|
self.assertIn('echo', handlers)
|
|
self.assertEqual(len(infos), 1)
|
|
info = infos[0]
|
|
self.assertEqual(info.name, 'echo')
|
|
self.assertIn('message', info.parameters.properties)
|
|
self.assertIn('message', info.parameters.required)
|
|
|
|
# 2. tools/call — handler returns the prefixed string
|
|
handler = handlers['echo']
|
|
call = ToolCall(id='c0', function=ToolFunction(name='echo', arguments={'message': 'hi'}))
|
|
observation = await handler(call, None)
|
|
self.assertIn('echoed: hi', observation)
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_filter_tools(self):
|
|
"""``tools=['echo']`` keeps the tool; an empty filter removes it."""
|
|
|
|
async def _run():
|
|
cfg = _echo_server_config()
|
|
cfg.tools = ['echo']
|
|
async with MCPServer(cfg) as server:
|
|
handlers, _ = await mcp_tools(server)
|
|
self.assertIn('echo', handlers)
|
|
|
|
cfg.tools = ['nonexistent']
|
|
async with MCPServer(cfg) as server:
|
|
handlers, _ = await mcp_tools(server)
|
|
self.assertNotIn('echo', handlers)
|
|
|
|
asyncio.run(_run())
|
|
|
|
def test_http_transport(self):
|
|
"""Streamable-HTTP transport: spawn echo server bound to a port, call ``echo``.
|
|
|
|
Regression test for the ``client.py`` HTTP path — older versions of
|
|
the mcp SDK accepted ``headers`` / ``timeout`` directly on
|
|
``streamable_http_client``; the current SDK requires those to go via
|
|
a caller-provided ``httpx.AsyncClient``. The stdio tests do not
|
|
exercise that code path.
|
|
"""
|
|
port = _free_port()
|
|
proc = subprocess.Popen(
|
|
[sys.executable, '-m', 'tests.agent.mcp_echo_server', 'http', str(port)],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
_wait_for_port(port)
|
|
|
|
cfg = MCPServerConfigHTTP(
|
|
url=f'http://127.0.0.1:{port}/mcp',
|
|
headers={'X-Test': 'evalscope'},
|
|
timeout=10.0,
|
|
)
|
|
|
|
async def _run():
|
|
async with MCPServer(cfg) as server:
|
|
handlers, infos = await mcp_tools(server)
|
|
self.assertIn('echo', handlers)
|
|
self.assertEqual(infos[0].name, 'echo')
|
|
|
|
call = ToolCall(
|
|
id='c0',
|
|
function=ToolFunction(name='echo', arguments={'message': 'via http'}),
|
|
)
|
|
observation = await handlers['echo'](call, None)
|
|
self.assertIn('echoed: via http', observation)
|
|
|
|
asyncio.run(_run())
|
|
finally:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|