13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
304 lines
12 KiB
Python
304 lines
12 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, patch
|
|
|
|
from evalscope.agent.tools.bash import BASH_TOOL_INFO
|
|
from evalscope.api.agent import NativeAgentConfig
|
|
from evalscope.api.dataset import Sample
|
|
from evalscope.api.evaluator import TaskState
|
|
from evalscope.api.metric import SampleScore, Score
|
|
from evalscope.api.model import ModelOutput
|
|
from evalscope.api.registry import get_benchmark
|
|
from evalscope.benchmarks.wide_search.utils import (
|
|
METRIC_NAMES,
|
|
WideSearchSession,
|
|
aggregate_official_scores,
|
|
date_near,
|
|
extract_number,
|
|
number_near,
|
|
url_match,
|
|
)
|
|
from evalscope.config import SandboxTaskConfig, TaskConfig
|
|
from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, JudgeStrategy
|
|
|
|
try:
|
|
import evalscope.agent.environments.enclave # noqa: F401
|
|
_ENCLAVE_AVAILABLE = True
|
|
except ImportError:
|
|
_ENCLAVE_AVAILABLE = False
|
|
|
|
|
|
def _evaluation(metric: str = 'exact_match') -> dict:
|
|
return {
|
|
'unique_columns': ['id'],
|
|
'required': ['id', 'value'],
|
|
'eval_pipeline': {
|
|
'id': {
|
|
'preprocess': ['norm_str'],
|
|
'metric': ['exact_match']
|
|
},
|
|
'value': {
|
|
'preprocess': ['norm_str'],
|
|
'metric': [metric],
|
|
'criterion': 'The values must have the same meaning.',
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
class TestWideSearchSession(unittest.TestCase):
|
|
|
|
def setUp(self) -> None:
|
|
self.gold = 'id,value\nA,one\nB,two\n'
|
|
|
|
def _score(self, prediction: str) -> tuple[dict, dict]:
|
|
session = WideSearchSession.create(prediction, self.gold, _evaluation())
|
|
return session.score({}, primary_key_maps={'id': {}})
|
|
|
|
def test_perfect_markdown_table(self) -> None:
|
|
prediction = '```markdown\n| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n```'
|
|
|
|
values, diagnostics = self._score(prediction)
|
|
|
|
self.assertEqual(diagnostics['matched_rows'], 2)
|
|
self.assertEqual(values, {name: 1.0 for name in METRIC_NAMES})
|
|
|
|
def test_row_and_item_metrics_handle_missing_extra_and_wrong_cells(self) -> None:
|
|
missing, _ = self._score('| id | value |\n| --- | --- |\n| A | one |')
|
|
extra, _ = self._score('| id | value |\n| --- | --- |\n| A | one |\n| B | two |\n| C | three |')
|
|
wrong, _ = self._score('| id | value |\n| --- | --- |\n| A | wrong |\n| B | two |')
|
|
|
|
self.assertEqual(missing['row_recall'], 0.5)
|
|
self.assertEqual(extra['row_precision'], 2 / 3)
|
|
self.assertEqual(wrong['row_precision'], 0.5)
|
|
self.assertEqual(wrong['item_precision'], 0.75)
|
|
|
|
def test_invalid_table_returns_zero_scores(self) -> None:
|
|
values, diagnostics = self._score('not a table')
|
|
|
|
self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES})
|
|
self.assertEqual(diagnostics['error'], 'response_df is None')
|
|
|
|
def test_duplicate_column_mapping_does_not_crash(self) -> None:
|
|
|
|
prediction = '| identifier | alias | value |\n| --- | --- | --- |\n| A | A | one |'
|
|
session = WideSearchSession.create(prediction, 'id,value\nA,one\n', _evaluation())
|
|
values, diagnostics = session.score({}, column_map={'identifier': 'id', 'alias': 'id'})
|
|
|
|
self.assertEqual(values, {name: 0.0 for name in METRIC_NAMES})
|
|
self.assertIn('required columns do not match', diagnostics['error'])
|
|
|
|
def test_empty_join_skips_column_judge(self) -> None:
|
|
|
|
prediction = '| id | value |\n| --- | --- |\n| B | candidate |'
|
|
session = WideSearchSession.create(prediction, 'id,value\nA,reference\n', _evaluation('llm_judge'))
|
|
inner, diagnostics = session.inner_frame(primary_key_maps={'id': {}})
|
|
|
|
self.assertTrue(inner.empty)
|
|
self.assertEqual(diagnostics['matched_rows'], 0)
|
|
|
|
def test_official_number_date_and_url_boundaries(self) -> None:
|
|
self.assertEqual(extract_number('about 1,234.5 kg'), '1234.5')
|
|
self.assertEqual(number_near('101', '100', 0.01), 1.0)
|
|
self.assertEqual(number_near('102', '100', 0.01), 0.0)
|
|
self.assertEqual(date_near('2025-02-01', '2025-01-01'), 1.0)
|
|
self.assertEqual(date_near('not a date', 'also invalid'), 1.0)
|
|
self.assertEqual(url_match('https://example.com/a', 'http://example.com/b'), 1.0)
|
|
|
|
|
|
class TestWideSearchAggregation(unittest.TestCase):
|
|
|
|
@staticmethod
|
|
def _sample_score(group_id: int, language: str, success: float, row_f1: float) -> SampleScore:
|
|
values = {name: row_f1 for name in METRIC_NAMES}
|
|
values['success_rate'] = success
|
|
return SampleScore(
|
|
score=Score(value=values, main_score_name='success_rate'),
|
|
sample_id=f'{language}-{group_id}',
|
|
group_id=group_id,
|
|
sample_metadata={'language': language},
|
|
)
|
|
|
|
def test_official_avg_pass_and_max_at_four(self) -> None:
|
|
scores = [self._sample_score(0, 'en', value, value * 0.8) for value in [0.0, 1.0, 0.0, 0.0]]
|
|
scores.extend(self._sample_score(1, 'zh', value, 0.2) for value in [0.0, 0.0, 0.0, 0.0])
|
|
|
|
aggregated = {(score.metric_name, score.aggregation, tuple(score.dimensions.items())): score.score
|
|
for score in aggregate_official_scores(scores)}
|
|
|
|
self.assertEqual(aggregated[('success_rate', 'mean', (('k', 4), ('scope', 'all')))], 0.125)
|
|
self.assertEqual(aggregated[('success_rate', 'pass_at_k', (('k', 4), ('scope', 'all')))], 0.5)
|
|
self.assertEqual(aggregated[('f1', 'max', (('k', 4), ('scope', 'en'), ('target', 'row')))], 0.8)
|
|
self.assertEqual(aggregated[('f1', 'mean', (('k', 4), ('scope', 'zh'), ('target', 'row')))], 0.2)
|
|
|
|
|
|
class TestWideSearchAdapter(unittest.TestCase):
|
|
|
|
@staticmethod
|
|
def _write_dataset(root: Path, count: int = 2) -> None:
|
|
(root / 'widesearch_gold').mkdir()
|
|
records = []
|
|
for index in range(count):
|
|
language = 'en' if index < count // 2 else 'zh'
|
|
instance_id = f'ws_{language}_{index + 1:03d}'
|
|
records.append({
|
|
'instance_id': instance_id,
|
|
'query': f'query-{language}-{index}',
|
|
'evaluation': json.dumps(_evaluation()),
|
|
'language': language,
|
|
})
|
|
(root / 'widesearch_gold' / f'{instance_id}.csv').write_text('\ufeffid,value\nA,one\n', encoding='utf-8')
|
|
(root / 'widesearch.jsonl').write_text('\n'.join(json.dumps(record) for record in records), encoding='utf-8')
|
|
|
|
def test_local_dataset_loads_repeats_and_strips_gold_bom(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
root = Path(tmp_dir)
|
|
self._write_dataset(root)
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
repeats=2,
|
|
dataset_args={'wide_search': {
|
|
'local_path': tmp_dir
|
|
}},
|
|
judge={'models': {'model_id': 'mock'}},
|
|
)
|
|
|
|
dataset = get_benchmark('wide_search', config=config).load_dataset()['default']
|
|
|
|
self.assertEqual(len(dataset), 4)
|
|
self.assertEqual([sample.group_id for sample in dataset], [0, 0, 1, 1])
|
|
self.assertEqual([sample.metadata['language'] for sample in dataset], ['en', 'en', 'zh', 'zh'])
|
|
self.assertTrue(all(sample.target.startswith('id,value') for sample in dataset))
|
|
|
|
def test_remote_dataset_downloads_full_snapshot(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
root = Path(tmp_dir)
|
|
self._write_dataset(root, count=1)
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
judge={'models': {'model_id': 'mock'}},
|
|
)
|
|
|
|
with patch(
|
|
'evalscope.api.dataset.hub.download_dataset_snapshot',
|
|
return_value=tmp_dir,
|
|
) as snapshot_download:
|
|
dataset = get_benchmark('wide_search', config=config).load_dataset()['default']
|
|
|
|
snapshot_download.assert_called_once_with(
|
|
data_id_or_path='bytedance-community/WideSearch',
|
|
data_source='modelscope',
|
|
revision=None,
|
|
force_redownload=False,
|
|
cache_dir=DEFAULT_DATASET_CACHE_DIR,
|
|
allow_file_pattern=None,
|
|
ignore_file_pattern=None,
|
|
)
|
|
self.assertEqual(len(dataset), 1)
|
|
self.assertEqual(dataset[0].metadata['instance_id'], 'ws_zh_001')
|
|
|
|
def test_base_metric_pipeline_uses_official_scorer(self) -> None:
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
judge={'strategy': JudgeStrategy.LLM, 'models': {'model_id': 'mock'}},
|
|
)
|
|
adapter = get_benchmark('wide_search', config=config)
|
|
adapter.llm_judge = Mock(generate=Mock(return_value=ModelOutput.from_content(model='mock', content='{"mapping": {}}')))
|
|
sample = Sample(
|
|
id=3,
|
|
group_id=2,
|
|
input='question',
|
|
target='id,value\nA,one\n',
|
|
metadata={
|
|
'instance_id': 'ws_en_001',
|
|
'language': 'en',
|
|
'evaluation': _evaluation(),
|
|
},
|
|
)
|
|
task_state = TaskState(
|
|
model='mock',
|
|
sample=sample,
|
|
output=ModelOutput.from_content(model='mock', content='| id | value |\n| --- | --- |\n| A | one |'),
|
|
completed=True,
|
|
)
|
|
|
|
sample_score = adapter.calculate_metrics(task_state)
|
|
|
|
self.assertEqual(sample_score.sample_id, 3)
|
|
self.assertEqual(sample_score.group_id, 2)
|
|
self.assertEqual(sample_score.score.value, {name: 1.0 for name in METRIC_NAMES})
|
|
|
|
def test_rule_only_judge_is_rejected(self) -> None:
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
judge={'strategy': JudgeStrategy.RULE, 'models': {'model_id': 'mock'}},
|
|
)
|
|
|
|
# ``get_benchmark`` is the only construction path, so the run fails before generation.
|
|
with self.assertRaisesRegex(ValueError, "judge.strategy='auto' or 'llm'"):
|
|
get_benchmark('wide_search', config=config)
|
|
|
|
@unittest.skipUnless(_ENCLAVE_AVAILABLE, 'ms_enclave (sandbox extra) is not installed')
|
|
def test_docker_uses_unified_sandbox_config(self) -> None:
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
sandbox=SandboxTaskConfig(
|
|
enabled=True,
|
|
default_config={
|
|
'image': 'custom:latest',
|
|
'network_enabled': False,
|
|
},
|
|
),
|
|
judge={'models': {'model_id': 'mock'}},
|
|
)
|
|
adapter = get_benchmark('wide_search', config=config)
|
|
sample = Sample(input='question', metadata={'instance_id': 'docker-test', 'language': 'en'})
|
|
|
|
with patch('evalscope.benchmarks.wide_search.wide_search_adapter.check_import'
|
|
), patch('evalscope.agent.environments.enclave.EnclaveAgentEnvironment') as environment_cls:
|
|
adapter.build_environment(sample)
|
|
|
|
environment_cls.assert_called_once_with(
|
|
engine='docker',
|
|
sandbox_config={
|
|
'image': 'custom:latest',
|
|
'network_enabled': False,
|
|
},
|
|
)
|
|
|
|
def test_official_prompts_function_calling_and_timeout_defaults(self) -> None:
|
|
config = TaskConfig(
|
|
model='mock',
|
|
datasets=['wide_search'],
|
|
agent_config=NativeAgentConfig(max_steps=7, command_timeout=12),
|
|
judge={'models': {'model_id': 'mock'}},
|
|
)
|
|
adapter = get_benchmark('wide_search', config=config)
|
|
sample = Sample(
|
|
input='question', tools=[BASH_TOOL_INFO], metadata={
|
|
'instance_id': 'ws_zh_001',
|
|
'language': 'zh'
|
|
}
|
|
)
|
|
|
|
self.assertEqual(adapter.build_strategy(sample).name, 'function_calling')
|
|
self.assertEqual(adapter._resolve_max_steps(config.agent_config), 7)
|
|
self.assertIn('联网信息搜索专家', adapter.build_initial_messages(sample)[0].content)
|
|
handlers, tools = adapter._resolve_tools(sample, config.agent_config)
|
|
default_handlers, default_tools = adapter._resolve_tools(sample, None)
|
|
self.assertIn('bash', handlers)
|
|
self.assertEqual(tools[0].parameters.properties['timeout'].default, 12)
|
|
self.assertEqual(default_tools[0].parameters.properties['timeout'].default, 120)
|
|
self.assertIn('bash', default_handlers)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|