import json import tempfile import unittest from pathlib import Path from unittest.mock import Mock, patch from evalscope.agent.tools.bash import BASH_TOOL_INFO from evalscope.api.agent import NativeAgentConfig from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState from evalscope.api.metric import SampleScore, Score from evalscope.api.model import ModelOutput from evalscope.api.registry import get_benchmark from evalscope.benchmarks.wide_search.utils import ( METRIC_NAMES, WideSearchSession, aggregate_official_scores, date_near, extract_number, number_near, url_match, ) from evalscope.config import SandboxTaskConfig, TaskConfig from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, JudgeStrategy try: import evalscope.agent.environments.enclave # noqa: F401 _ENCLAVE_AVAILABLE = True except ImportError: _ENCLAVE_AVAILABLE = False def _evaluation(metric: str = 'exact_match') -> dict: return { 'unique_columns': ['id'], 'required': ['id', 'value'], 'eval_pipeline': { 'id': { 'preprocess': ['norm_str'], 'metric': ['exact_match'] }, 'value': { 'preprocess': ['norm_str'], 'metric': [metric], 'criterion': 'The values must have the same meaning.', }, }, } class TestWideSearchSession(unittest.TestCase): def setUp(self) -> None: self.gold = 'id,value\nA,one\nB,two\n' def _score(self, prediction: str) -> tuple[dict, dict]: session = WideSearchSession.create(prediction, self.gold, _evaluation()) return session.score({}, primary_key_maps={'id': {}}) def test_perfect_markdown_table(self) -> None: prediction = '```markdown\n| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n```' values, diagnostics = self._score(prediction) self.assertEqual(diagnostics['matched_rows'], 2) self.assertEqual(values, {name: 1.0 for name in METRIC_NAMES}) def test_row_and_item_metrics_handle_missing_extra_and_wrong_cells(self) -> None: missing, _ = self._score('| id | value |\n| --- | --- |\n| A | one |') extra, _ = self._score('| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n| C | three |') wrong, _ = self._score('| id | value |\n| --- | --- |\n| A | wrong |\n| B | two |') self.assertEqual(missing['row_recall'], 0.5) self.assertEqual(extra['row_precision'], 2 / 3) self.assertEqual(wrong['row_precision'], 0.5) self.assertEqual(wrong['item_precision'], 0.75) def test_invalid_table_returns_zero_scores(self) -> None: values, diagnostics = self._score('not a table') self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES}) self.assertEqual(diagnostics['error'], 'response_df is None') def test_duplicate_column_mapping_does_not_crash(self) -> None: prediction = '| identifier | alias | value |\n| --- | --- | --- |\n| A | A | one |' session = WideSearchSession.create(prediction, 'id,value\nA,one\n', _evaluation()) values, diagnostics = session.score({}, column_map={'identifier': 'id', 'alias': 'id'}) self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES}) self.assertIn('required columns do not match', diagnostics['error']) def test_empty_join_skips_column_judge(self) -> None: prediction = '| id | value |\n| --- | --- |\n| B | candidate |' session = WideSearchSession.create(prediction, 'id,value\nA,reference\n', _evaluation('llm_judge')) inner, diagnostics = session.inner_frame(primary_key_maps={'id': {}}) self.assertTrue(inner.empty) self.assertEqual(diagnostics['matched_rows'], 0) def test_official_number_date_and_url_boundaries(self) -> None: self.assertEqual(extract_number('about 1,234.5 kg'), '1234.5') self.assertEqual(number_near('101', '100', 0.01), 1.0) self.assertEqual(number_near('102', '100', 0.01), 0.0) self.assertEqual(date_near('2025-02-01', '2025-01-01'), 1.0) self.assertEqual(date_near('not a date', 'also invalid'), 1.0) self.assertEqual(url_match('https://example.com/a', 'http://example.com/b'), 1.0) class TestWideSearchAggregation(unittest.TestCase): @staticmethod def _sample_score(group_id: int, language: str, success: float, row_f1: float) -> SampleScore: values = {name: row_f1 for name in METRIC_NAMES} values['success_rate'] = success return SampleScore( score=Score(value=values, main_score_name='success_rate'), sample_id=f'{language}-{group_id}', group_id=group_id, sample_metadata={'language': language}, ) def test_official_avg_pass_and_max_at_four(self) -> None: scores = [self._sample_score(0, 'en', value, value * 0.8) for value in [0.0, 1.0, 0.0, 0.0]] scores.extend(self._sample_score(1, 'zh', value, 0.2) for value in [0.0, 0.0, 0.0, 0.0]) aggregated = {(score.metric_name, score.aggregation, tuple(score.dimensions.items())): score.score for score in aggregate_official_scores(scores)} self.assertEqual(aggregated[('success_rate', 'mean', (('k', 4), ('scope', 'all')))], 0.125) self.assertEqual(aggregated[('success_rate', 'pass_at_k', (('k', 4), ('scope', 'all')))], 0.5) self.assertEqual(aggregated[('f1', 'max', (('k', 4), ('scope', 'en'), ('target', 'row')))], 0.8) self.assertEqual(aggregated[('f1', 'mean', (('k', 4), ('scope', 'zh'), ('target', 'row')))], 0.2) class TestWideSearchAdapter(unittest.TestCase): @staticmethod def _write_dataset(root: Path, count: int = 2) -> None: (root / 'widesearch_gold').mkdir() records = [] for index in range(count): language = 'en' if index < count // 2 else 'zh' instance_id = f'ws_{language}_{index + 1:03d}' records.append({ 'instance_id': instance_id, 'query': f'query-{language}-{index}', 'evaluation': json.dumps(_evaluation()), 'language': language, }) (root / 'widesearch_gold' / f'{instance_id}.csv').write_text('\ufeffid,value\nA,one\n', encoding='utf-8') (root / 'widesearch.jsonl').write_text('\n'.join(json.dumps(record) for record in records), encoding='utf-8') def test_local_dataset_loads_repeats_and_strips_gold_bom(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir) self._write_dataset(root) config = TaskConfig( model='mock', datasets=['wide_search'], repeats=2, dataset_args={'wide_search': { 'local_path': tmp_dir }}, judge={'models': {'model_id': 'mock'}}, ) dataset = get_benchmark('wide_search', config=config).load_dataset()['default'] self.assertEqual(len(dataset), 4) self.assertEqual([sample.group_id for sample in dataset], [0, 0, 1, 1]) self.assertEqual([sample.metadata['language'] for sample in dataset], ['en', 'en', 'zh', 'zh']) self.assertTrue(all(sample.target.startswith('id,value') for sample in dataset)) def test_remote_dataset_downloads_full_snapshot(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir) self._write_dataset(root, count=1) config = TaskConfig( model='mock', datasets=['wide_search'], judge={'models': {'model_id': 'mock'}}, ) with patch( 'evalscope.api.dataset.hub.download_dataset_snapshot', return_value=tmp_dir, ) as snapshot_download: dataset = get_benchmark('wide_search', config=config).load_dataset()['default'] snapshot_download.assert_called_once_with( data_id_or_path='bytedance-community/WideSearch', data_source='modelscope', revision=None, force_redownload=False, cache_dir=DEFAULT_DATASET_CACHE_DIR, allow_file_pattern=None, ignore_file_pattern=None, ) self.assertEqual(len(dataset), 1) self.assertEqual(dataset[0].metadata['instance_id'], 'ws_zh_001') def test_base_metric_pipeline_uses_official_scorer(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], judge={'strategy': JudgeStrategy.LLM, 'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) adapter.llm_judge = Mock(generate=Mock(return_value=ModelOutput.from_content(model='mock', content='{"mapping": {}}'))) sample = Sample( id=3, group_id=2, input='question', target='id,value\nA,one\n', metadata={ 'instance_id': 'ws_en_001', 'language': 'en', 'evaluation': _evaluation(), }, ) task_state = TaskState( model='mock', sample=sample, output=ModelOutput.from_content(model='mock', content='| id | value |\n| --- | --- |\n| A | one |'), completed=True, ) sample_score = adapter.calculate_metrics(task_state) self.assertEqual(sample_score.sample_id, 3) self.assertEqual(sample_score.group_id, 2) self.assertEqual(sample_score.score.value, {name: 1.0 for name in METRIC_NAMES}) def test_rule_only_judge_is_rejected(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], judge={'strategy': JudgeStrategy.RULE, 'models': {'model_id': 'mock'}}, ) # ``get_benchmark`` is the only construction path, so the run fails before generation. with self.assertRaisesRegex(ValueError, "judge.strategy='auto' or 'llm'"): get_benchmark('wide_search', config=config) @unittest.skipUnless(_ENCLAVE_AVAILABLE, 'ms_enclave (sandbox extra) is not installed') def test_docker_uses_unified_sandbox_config(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], sandbox=SandboxTaskConfig( enabled=True, default_config={ 'image': 'custom:latest', 'network_enabled': False, }, ), judge={'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) sample = Sample(input='question', metadata={'instance_id': 'docker-test', 'language': 'en'}) with patch('evalscope.benchmarks.wide_search.wide_search_adapter.check_import' ), patch('evalscope.agent.environments.enclave.EnclaveAgentEnvironment') as environment_cls: adapter.build_environment(sample) environment_cls.assert_called_once_with( engine='docker', sandbox_config={ 'image': 'custom:latest', 'network_enabled': False, }, ) def test_official_prompts_function_calling_and_timeout_defaults(self) -> None: config = TaskConfig( model='mock', datasets=['wide_search'], agent_config=NativeAgentConfig(max_steps=7, command_timeout=12), judge={'models': {'model_id': 'mock'}}, ) adapter = get_benchmark('wide_search', config=config) sample = Sample( input='question', tools=[BASH_TOOL_INFO], metadata={ 'instance_id': 'ws_zh_001', 'language': 'zh' } ) self.assertEqual(adapter.build_strategy(sample).name, 'function_calling') self.assertEqual(adapter._resolve_max_steps(config.agent_config), 7) self.assertIn('联网信息搜索专家', adapter.build_initial_messages(sample)[0].content) handlers, tools = adapter._resolve_tools(sample, config.agent_config) default_handlers, default_tools = adapter._resolve_tools(sample, None) self.assertIn('bash', handlers) self.assertEqual(tools[0].parameters.properties['timeout'].default, 12) self.assertEqual(default_tools[0].parameters.properties['timeout'].default, 120) self.assertIn('bash', default_handlers) if __name__ == '__main__': unittest.main()