import asyncio import csv import json import pytest from pathlib import Path from typing import Any, Dict, List from evalscope.api.benchmark import BenchmarkMeta from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState from evalscope.api.metric import SampleScore, Score from evalscope.api.registry import get_benchmark from evalscope.api.tool import ToolCall, ToolFunction from evalscope.benchmarks.mcp_atlas.mcp_atlas_adapter import MCPAtlasAdapter from evalscope.benchmarks.mcp_atlas.utils import ( MCPAtlasClient, MCPAtlasServerUnavailable, extract_claims, extract_required_servers, is_transport_error, mcp_tool_to_tool_info, parse_claim_judge_response, parse_enabled_tools, ) from evalscope.config import TaskConfig from evalscope.constants import HubType class FakeClient: def __init__(self) -> None: self.calls: List[Dict[str, Any]] = [] def enabled_servers(self) -> List[str]: return ['wikipedia'] def list_tools(self) -> List[Dict[str, Any]]: return [ { 'name': 'wikipedia_get_article', 'description': 'Fetch a Wikipedia article.', 'inputSchema': { 'type': 'object', 'properties': { 'title': { 'type': 'string', 'description': 'Article title.', } }, 'required': ['title'], }, }, { 'name': 'github_search_repositories', 'description': 'Search repositories.', 'inputSchema': { 'type': 'object', 'properties': {}, }, }, ] def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> str: self.calls.append({'tool_name': tool_name, 'tool_args': tool_args}) return 'tool result' class PromptJudge: model_id = 'judge-model' def judge(self, prompt: str) -> str: outcome = 'not_fulfilled' if 'claim one' in prompt: outcome = 'fulfilled' elif 'claim two' in prompt: outcome = 'partially_fulfilled' return json.dumps({ 'coverage_outcome': outcome, 'justification': outcome, 'confidence_level': 0.9, }) def make_adapter(limit: Any = None, local_path: str = '', **extra_params: Any) -> MCPAtlasAdapter: base_extra_params = { 'mcp_server_url': 'http://localhost:1984', 'filter_enabled_servers': True, 'max_steps': 100, 'max_tool_calls': 100, 'request_timeout': 60.0, 'list_tools_timeout': 180.0, 'use_system_prompt': False, 'pass_threshold': 0.75, } base_extra_params.update(extra_params) meta = BenchmarkMeta( name='mcp_atlas', dataset_id='ScaleAI/MCP-Atlas', subset_list=['default'], default_subset='default', eval_split='train', prompt_template='{question}', metric_list=['coverage_score', 'pass'], extra_params=base_extra_params, ) dataset_args = {'extra_params': extra_params} if local_path: dataset_args['local_path'] = local_path meta._update({'local_path': local_path}) cfg = TaskConfig( datasets=['mcp_atlas'], dataset_args={'mcp_atlas': dataset_args}, dataset_hub=HubType.MODELSCOPE, limit=limit, ) return MCPAtlasAdapter(benchmark_meta=meta, task_config=cfg) def write_rows(path: Path, rows: List[Dict[str, Any]]) -> None: with path.open('w', newline='', encoding='utf-8') as f: writer = csv.DictWriter(f, fieldnames=['TASK', 'PROMPT', 'ENABLED_TOOLS', 'TRAJECTORY', 'GTFA_CLAIMS']) writer.writeheader() writer.writerows(rows) def test_mcp_atlas_registered_under_short_name() -> None: cfg = TaskConfig(datasets=['mcp_atlas']) adapter = get_benchmark('mcp_atlas', cfg) assert isinstance(adapter, MCPAtlasAdapter) assert adapter.name == 'mcp_atlas' def test_parse_enabled_tools_accepts_strings_and_objects() -> None: tools = parse_enabled_tools( json.dumps([ 'wikipedia_get_article', { 'name': 'github_search_repositories' }, 'wikipedia_get_article', 123, ]) ) assert tools == ['wikipedia_get_article', 'github_search_repositories'] def test_extract_required_servers_uses_trajectory_tool_calls() -> None: trajectory = json.dumps([{ 'role': 'assistant', 'tool_calls': [ { 'function': { 'name': 'wikipedia_get_article' } }, { 'function': { 'name': 'MongoDB_find' } }, { 'function': { 'name': 'brave_brave_web_search' } }, ], }]) assert extract_required_servers(trajectory) == ['brave-search', 'mongodb', 'wikipedia'] def test_record_to_sample_sets_metadata_and_claims() -> None: adapter = make_adapter() adapter._client = FakeClient() sample = adapter.record_to_sample({ 'TASK': 'task-1', 'PROMPT': 'Find the answer.', 'ENABLED_TOOLS': json.dumps(['wikipedia_get_article']), 'TRAJECTORY': json.dumps([]), 'GTFA_CLAIMS': json.dumps([{ 'claim': 'The answer cites the page.' }]), }) assert sample.input == 'Find the answer.' assert extract_claims(sample.target) == ['The answer cites the page.'] assert sample.metadata['task_id'] == 'task-1' assert sample.metadata['enabled_tools'] == ['wikipedia_get_article'] assert sample.tools[0].name == 'wikipedia_get_article' def test_extract_claims_accepts_python_list_literal() -> None: claims = extract_claims("['claim one', 'claim two']") assert claims == ['claim one', 'claim two'] def test_extract_claims_flattens_cached_single_literal_list() -> None: claims = extract_claims(json.dumps(["['claim one', 'claim two']"])) assert claims == ['claim one', 'claim two'] def test_tool_info_conversion_accepts_input_schema() -> None: tool_info = mcp_tool_to_tool_info({ 'name': 'wikipedia_get_article', 'description': 'Fetch a Wikipedia article.', 'inputSchema': { 'type': 'object', 'properties': { 'title': { 'type': 'string', 'description': 'Article title.', } }, 'required': ['title'], }, }) assert tool_info.name == 'wikipedia_get_article' assert tool_info.parameters.required == ['title'] assert tool_info.parameters.properties['title'].type == 'string' def test_client_uses_mcp_atlas_http_endpoints(monkeypatch: Any) -> None: requests_seen: List[Dict[str, Any]] = [] class Response: def __init__(self, status_code: int, payload: Any) -> None: self.status_code = status_code self.payload = payload self.text = json.dumps(payload) def raise_for_status(self) -> None: pass def json(self) -> Any: return self.payload def fake_get(url: str, timeout: float) -> Response: requests_seen.append({'method': 'GET', 'url': url, 'timeout': timeout}) return Response(200, {'enabled_servers': ['wikipedia']}) def fake_post(url: str, **kwargs: Any) -> Response: requests_seen.append({'method': 'POST', 'url': url, **kwargs}) if url.endswith('/list-tools'): return Response(200, [{'name': 'wikipedia_get_article'}]) if url.endswith('/call-tool'): return Response(200, [{'type': 'text', 'text': 'article text'}]) raise AssertionError(f'Unexpected URL: {url}') monkeypatch.setattr('requests.get', fake_get) monkeypatch.setattr('requests.post', fake_post) client = MCPAtlasClient('http://localhost:1984/', request_timeout=7.0, list_tools_timeout=11.0) assert client.enabled_servers() == ['wikipedia'] assert client.list_tools() == [{'name': 'wikipedia_get_article'}] assert client.call_tool('wikipedia_get_article', {'title': 'MCP'}) == 'article text' assert requests_seen[0] == { 'method': 'GET', 'url': 'http://localhost:1984/enabled-servers', 'timeout': 11.0, } assert requests_seen[1]['url'] == 'http://localhost:1984/list-tools' assert requests_seen[2]['json'] == { 'tool_name': 'wikipedia_get_article', 'tool_args': { 'title': 'MCP' }, } def test_client_accepts_servers_dict_response(monkeypatch: Any) -> None: class Response: def raise_for_status(self) -> None: pass def json(self) -> Dict[str, Any]: return { 'servers': { 'wikipedia': 'OK', 'github': 'ERROR_NOT_ONLINE', } } def fake_get(url: str, timeout: float) -> Response: return Response() monkeypatch.setattr('requests.get', fake_get) client = MCPAtlasClient('http://localhost:1984', request_timeout=7.0, list_tools_timeout=11.0) assert client.enabled_servers() == ['wikipedia'] def test_client_marks_transport_500_as_unavailable(monkeypatch: Any) -> None: class Response: status_code = 500 text = '{"detail":"connect ECONNREFUSED 199.193.116.105:443"}' def json(self) -> Any: return {} def fake_post(url: str, **kwargs: Any) -> Response: return Response() monkeypatch.setattr('requests.post', fake_post) client = MCPAtlasClient('http://localhost:1984', request_timeout=7.0, list_tools_timeout=11.0) with pytest.raises(MCPAtlasServerUnavailable) as exc_info: client.call_tool('open-library_get_book_by_title', {'title': 'The Sins of the Wolf'}) assert exc_info.value.server_name == 'open-library' assert is_transport_error(str(exc_info.value)) def test_client_returns_bounded_http_error(monkeypatch: Any) -> None: class Response: status_code = 500 text = 'x' * 1200 def json(self) -> Any: return {} def fake_post(url: str, **kwargs: Any) -> Response: return Response() monkeypatch.setattr('requests.post', fake_post) client = MCPAtlasClient('http://localhost:1984', request_timeout=7.0, list_tools_timeout=11.0) result = client.call_tool('wikipedia_get_article', {'title': 'MCP'}) assert result.startswith('Error calling tool wikipedia_get_article (HTTP 500): ') assert result.endswith('...') assert len(result) < 1100 def test_client_handles_malformed_json_response(monkeypatch: Any) -> None: class Response: status_code = 200 text = '{not json' def json(self) -> Any: raise ValueError('bad json') def fake_post(url: str, **kwargs: Any) -> Response: return Response() monkeypatch.setattr('requests.post', fake_post) client = MCPAtlasClient('http://localhost:1984', request_timeout=7.0, list_tools_timeout=11.0) result = client.call_tool('wikipedia_get_article', {'title': 'MCP'}) assert result == 'Error decoding tool response JSON from wikipedia_get_article: bad json. Raw response: {not json' def test_load_dataset_filters_missing_servers_and_attaches_tools(tmp_path: Path) -> None: dataset_path = tmp_path / 'mcp_atlas.csv' write_rows( dataset_path, [ { 'TASK': 'keep', 'PROMPT': 'Use Wikipedia.', 'ENABLED_TOOLS': json.dumps(['wikipedia_get_article']), 'TRAJECTORY': json.dumps([{ 'tool_calls': [{ 'function': { 'name': 'wikipedia_get_article' } }] }]), 'GTFA_CLAIMS': json.dumps(['Uses the Wikipedia result.']), }, { 'TASK': 'drop', 'PROMPT': 'Use GitHub.', 'ENABLED_TOOLS': json.dumps(['github_search_repositories']), 'TRAJECTORY': json.dumps([{ 'tool_calls': [{ 'function': { 'name': 'github_search_repositories' } }] }]), 'GTFA_CLAIMS': json.dumps(['Uses the GitHub result.']), }, ] ) adapter = make_adapter(local_path=str(tmp_path)) adapter._client = FakeClient() dataset = adapter.load_dataset()['default'] assert len(dataset) == 1 assert dataset[0].metadata['task_id'] == 'keep' assert dataset[0].tools[0].name == 'wikipedia_get_article' assert adapter._excluded_tasks == [{'task_id': 'drop', 'missing_servers': ['github']}] def test_sample_filter_logs_missing_servers(monkeypatch: Any) -> None: logs: List[Dict[str, Any]] = [] def fake_warning(message: str, task_id: str, missing: List[str]) -> None: logs.append({ 'message': message, 'task_id': task_id, 'missing': missing, }) monkeypatch.setattr('evalscope.benchmarks.mcp_atlas.mcp_atlas_adapter.logger.warning', fake_warning) adapter = make_adapter() adapter._enabled_servers = ['wikipedia'] sample = Sample(input='prompt', metadata={'task_id': 'drop', 'required_servers': ['github']}) assert adapter.sample_filter(sample) is False assert adapter._excluded_tasks == [{'task_id': 'drop', 'missing_servers': ['github']}] assert logs == [{ 'message': 'Skipping MCP-Atlas task %s because required servers are not enabled: %s', 'task_id': 'drop', 'missing': ['github'], }] def test_build_tools_enforces_tool_call_limit() -> None: adapter = make_adapter(max_tool_calls=1) fake_client = FakeClient() adapter._client = fake_client sample = Sample( input='prompt', id=3, metadata={'enabled_tools': ['wikipedia_get_article']}, ) handler = adapter.build_tools(sample)['wikipedia_get_article'] call = ToolCall( id='call-1', function=ToolFunction(name='wikipedia_get_article', arguments={'title': 'MCP'}), ) first = asyncio.run(handler(call, None)) second = asyncio.run(handler(call, None)) assert first == 'tool result' assert second == 'MCP-Atlas tool call limit exceeded (1).' assert fake_client.calls == [{'tool_name': 'wikipedia_get_article', 'tool_args': {'title': 'MCP'}}] def test_build_tools_short_circuits_failed_server() -> None: class FailingClient(FakeClient): def call_tool(self, tool_name: str, tool_args: Dict[str, Any]) -> str: self.calls.append({'tool_name': tool_name, 'tool_args': tool_args}) raise MCPAtlasServerUnavailable(tool_name, 'connect ECONNREFUSED 199.193.116.105:443') adapter = make_adapter() fake_client = FailingClient() adapter._client = fake_client sample = Sample( input='prompt', id=4, metadata={'enabled_tools': ['open-library_get_book_by_title', 'open-library_get_authors_by_name']}, ) handlers = adapter.build_tools(sample) first_call = ToolCall( id='call-1', function=ToolFunction(name='open-library_get_book_by_title', arguments={'title': 'The Sins of the Wolf'}), ) second_call = ToolCall( id='call-2', function=ToolFunction(name='open-library_get_authors_by_name', arguments={'name': 'Anne Perry'}), ) first = asyncio.run(handlers['open-library_get_book_by_title'](first_call, None)) second = asyncio.run(handlers['open-library_get_authors_by_name'](second_call, None)) assert "MCP server 'open-library' is unavailable" in first assert "MCP server 'open-library' is unavailable" in second assert fake_client.calls == [{ 'tool_name': 'open-library_get_book_by_title', 'tool_args': { 'title': 'The Sins of the Wolf' }, }] def test_llm_match_score_computes_claim_coverage() -> None: adapter = make_adapter(pass_threshold=0.75) adapter.llm_judge = PromptJudge() sample = Sample( input='question', target=json.dumps(['claim one', 'claim two', 'claim three']), id=0, ) state = TaskState(model='mock', sample=sample, completed=True) score = adapter.llm_match_score( original_prediction='answer', filtered_prediction='answer', reference=sample.target, task_state=state, ) assert score.value['coverage_score'] == pytest.approx(0.5) assert score.value['pass'] == 0.0 assert score.metadata['fully_covered_claims'] == 1 assert score.metadata['partially_covered_claims'] == 1 def test_parse_claim_judge_response_accepts_string_confidence() -> None: outcome, justification, confidence = parse_claim_judge_response( json.dumps({ 'coverage_outcome': 'fulfilled', 'justification': 'covered', 'confidence_level': 'high', }) ) assert outcome == 'fulfilled' assert justification == 'covered' assert confidence == 1.0 def test_parse_claim_judge_response_accepts_fallback_keys() -> None: outcome, justification, confidence = parse_claim_judge_response( json.dumps({ 'outcome': 'partially_fulfilled', 'reason': 'partly covered', 'confidence': 'medium', }) ) assert outcome == 'partially_fulfilled' assert justification == 'partly covered' assert confidence == 0.5 def test_aggregate_scores_returns_mean_coverage_and_pass_rate() -> None: adapter = make_adapter() scores = adapter.aggregate_scores([ SampleScore(sample_id=0, score=Score(value={ 'coverage_score': 1.0, 'pass': 1.0 })), SampleScore(sample_id=1, score=Score(value={ 'coverage_score': 0.5, 'pass': 0.0 })), ]) assert [(score.metric_name, score.score) for score in scores] == [ ('coverage_score', 0.75), ('pass_rate', 0.5), ]