234 lines
8.5 KiB
Python
234 lines
8.5 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
import json
|
|
import tempfile
|
|
from dotenv import dotenv_values, load_dotenv
|
|
from pathlib import Path
|
|
|
|
load_dotenv('.env')
|
|
|
|
env = dotenv_values('.env')
|
|
|
|
import unittest
|
|
|
|
from evalscope.config import SandboxTaskConfig
|
|
from evalscope.constants import EvalType, JudgeStrategy, OutputType
|
|
from evalscope.utils.logger import get_logger
|
|
from tests.common import TestBenchmark
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
class TestAgentBenchmark(TestBenchmark):
|
|
"""Agentic benchmark evaluation test cases."""
|
|
|
|
def setUp(self):
|
|
"""Setup common test configuration."""
|
|
self.base_config = {
|
|
'model': 'qwen3-max',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'eval_type': EvalType.OPENAI_API,
|
|
'eval_batch_size': 5,
|
|
'limit': 5,
|
|
'generation_config': {
|
|
'temperature': 0.7,
|
|
'parallel_tool_calls': True,
|
|
'retries': 3,
|
|
'extra_body': {'enable_thinking': True},
|
|
'stream': True
|
|
},
|
|
'judge_strategy': JudgeStrategy.AUTO,
|
|
'judge_model_args': {
|
|
'model_id': 'qwen3-max',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'generation_config': {
|
|
'temperature': 0.0,
|
|
'extra_body': {'enable_thinking': False}
|
|
}
|
|
},
|
|
'debug': True,
|
|
}
|
|
|
|
def test_browsecomp(self):
|
|
"""Test BrowseComp benchmark end-to-end."""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
config_overrides = {
|
|
'collect_perf': False,
|
|
'debug': False,
|
|
'eval_batch_size': 1,
|
|
'limit': 1,
|
|
'no_timestamp': True,
|
|
'work_dir': tmp_dir,
|
|
}
|
|
if not env.get('DASHSCOPE_API_KEY'):
|
|
config_overrides['judge_strategy'] = JudgeStrategy.RULE
|
|
|
|
self._run_dataset_test('browsecomp', **config_overrides)
|
|
|
|
review_files = list(Path(tmp_dir).glob('reviews/*/browsecomp_default.jsonl'))
|
|
self.assertEqual(len(review_files), 1)
|
|
review = json.loads(review_files[0].read_text(encoding='utf-8').strip())
|
|
self.assertNotIn('canary', review['sample_score']['sample_metadata'])
|
|
|
|
def test_swe_bench_verified_agentic(self):
|
|
"""Test SWE-bench-verified agentic dataset using docker environment."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'toolcall',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'build_docker_images': True,
|
|
'pull_remote_images_if_available': True,
|
|
'force_arch': 'arm64',
|
|
}
|
|
}
|
|
self._run_dataset_test('swe_bench_verified_agentic', dataset_args, limit=1)
|
|
|
|
def test_swe_bench_verified_mini_agentic(self):
|
|
"""Test SWE-bench-verified-mini agentic dataset using docker environment."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'toolcall',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'build_docker_images': True,
|
|
'pull_remote_images_if_available': True,
|
|
'force_arch': 'arm64',
|
|
}
|
|
}
|
|
self._run_dataset_test('swe_bench_verified_mini_agentic', dataset_args, limit=3)
|
|
|
|
def test_swe_bench_lite_agentic(self):
|
|
"""Test SWE-bench-lite agentic dataset using docker environment."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'toolcall',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'build_docker_images': True,
|
|
'pull_remote_images_if_available': True,
|
|
'force_arch': 'arm64',
|
|
}
|
|
}
|
|
self._run_dataset_test('swe_bench_lite_agentic', dataset_args, limit=1)
|
|
|
|
def test_swe_bench_multilingual_agentic(self):
|
|
"""Test SWE-bench-multilingual agentic dataset using docker environment."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'toolcall',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'build_docker_images': False,
|
|
'pull_remote_images_if_available': True,
|
|
}
|
|
}
|
|
self._run_dataset_test(
|
|
'swe_bench_multilingual_agentic',
|
|
dataset_args,
|
|
limit=1,
|
|
generation_config={
|
|
'temperature': 0.0,
|
|
'parallel_tool_calls': False,
|
|
'retries': 3,
|
|
'extra_body': {'enable_thinking': True},
|
|
'stream': True
|
|
},
|
|
)
|
|
|
|
def test_swe_bench_pro(self):
|
|
"""Test SWE-bench_Pro agentic dataset using docker environment."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'toolcall',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'eval_timeout': 1800,
|
|
}
|
|
}
|
|
self._run_dataset_test(
|
|
'swe_bench_pro',
|
|
dataset_args,
|
|
limit=5,
|
|
use_cache='outputs/20260519_155200',
|
|
rerun_review=True,
|
|
sandbox=SandboxTaskConfig(
|
|
default_config={'platform': 'linux/amd64', 'memory_limit': '12g', 'cpu_limit': 4.0},
|
|
),
|
|
)
|
|
|
|
def test_gaia(self):
|
|
"""Test GAIA benchmark using docker environment with react + bash."""
|
|
dataset_args = {
|
|
'subset_list': ['2023_level1', '2023_level2', '2023_level3'],
|
|
'extra_params': {
|
|
'max_steps': 50,
|
|
'command_timeout': 180.0,
|
|
'docker_image': 'python:3.11',
|
|
'network_enabled': True,
|
|
}
|
|
}
|
|
self._run_dataset_test('gaia', dataset_args, limit=1)
|
|
|
|
def test_gaia_with_mcp(self):
|
|
"""GAIA + MCP fetch server, exercising the host-side MCP plumbing.
|
|
|
|
Requires ``pip install mcp-server-fetch`` in the eval environment.
|
|
Using ``python -m mcp_server_fetch`` (rather than ``uvx``) keeps the
|
|
test deterministic — no per-run package fetch / venv creation.
|
|
"""
|
|
import sys
|
|
|
|
from evalscope.api.agent import NativeAgentConfig
|
|
from evalscope.api.agent.mcp import MCPServerConfigStdio
|
|
dataset_args = {
|
|
'subset_list': ['2023_level1'],
|
|
'extra_params': {
|
|
'max_steps': 30,
|
|
'command_timeout': 180.0,
|
|
'docker_image': 'python:3.11',
|
|
'network_enabled': True,
|
|
}
|
|
}
|
|
agent_config = NativeAgentConfig(mcp_servers=[
|
|
MCPServerConfigStdio(
|
|
command=sys.executable,
|
|
# ``--ignore-robots-txt`` lets the server fetch sites whose
|
|
# robots.txt is unreachable (transient network failures /
|
|
# CDN-blocked UAs commonly seen during offline-ish CI runs).
|
|
args=['-m', 'mcp_server_fetch', '--ignore-robots-txt'],
|
|
name='fetch',
|
|
),
|
|
])
|
|
self._run_dataset_test('gaia', dataset_args, limit=1, agent_config=agent_config)
|
|
|
|
def test_terminal_bench_v2_1(self):
|
|
"""Test Terminal-Bench v2.1 dataset."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'timeout_multiplier': 3,
|
|
'environment_kwargs': {'override_cpus': 2},
|
|
},
|
|
}
|
|
self._run_dataset_test('terminal_bench_v2_1', dataset_args, limit=3, eval_batch_size=3)
|
|
|
|
def test_swe_bench_verified_agentic_backticks(self):
|
|
"""Test SWE-bench-verified agentic dataset with backticks protocol."""
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'action_protocol': 'backticks',
|
|
'max_steps': 250,
|
|
'command_timeout': 60.0,
|
|
'build_docker_images': True,
|
|
'pull_remote_images_if_available': True,
|
|
'force_arch': 'arm64',
|
|
}
|
|
}
|
|
self._run_dataset_test('swe_bench_verified_agentic', dataset_args, limit=1)
|
|
|
|
if __name__ == '__main__':
|
|
# Run specific test: python -m unittest test_agent.TestAgentBenchmark.test_swe_bench_verified_agentic
|
|
# Run all tests: python -m unittest test_agent.TestAgentBenchmark
|
|
unittest.main()
|