13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
97 lines
2.7 KiB
Python
97 lines
2.7 KiB
Python
"""Tests for benchmark metadata validation."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from evalscope.api.benchmark import BenchmarkMeta
|
|
from evalscope.api.metric.semantics import MetricSelector
|
|
from evalscope.utils.doc_utils.generate_dataset_md import extract_benchmark_meta
|
|
|
|
|
|
def test_runtime_update_revalidates_primary_metric() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='multi_metric',
|
|
dataset_id='local',
|
|
metric_list=['accuracy', 'f1_score'],
|
|
primary_metric='accuracy',
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="primary_metric='missing'"):
|
|
meta._update({'primary_metric': 'missing'})
|
|
|
|
|
|
def test_string_primary_metric_is_first_class_shorthand() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='single_name_selector',
|
|
dataset_id='local',
|
|
metric_list=['accuracy'],
|
|
primary_metric='accuracy',
|
|
)
|
|
|
|
assert meta.primary_metric == MetricSelector(name='accuracy')
|
|
|
|
|
|
def test_legacy_metric_list_aliases_are_normalized_at_the_adapter_boundary() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='legacy_adapter',
|
|
dataset_id='local',
|
|
metric_list=['acc', 'f1_score'],
|
|
primary_metric=MetricSelector(name='accuracy'),
|
|
)
|
|
|
|
assert meta.metric_list == ['accuracy', 'f1']
|
|
|
|
|
|
def test_runtime_metric_list_update_revalidates_primary_metric() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='multi_metric',
|
|
dataset_id='local',
|
|
metric_list=['accuracy', 'f1_score'],
|
|
primary_metric='accuracy',
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="primary_metric='accuracy'"):
|
|
meta._update({'metric_list': ['f1_score']})
|
|
|
|
|
|
def test_doc_metadata_extraction_does_not_instantiate_adapter() -> None:
|
|
|
|
class RuntimeOnlyAdapter:
|
|
|
|
def __init__(self) -> None:
|
|
raise AssertionError('documentation metadata must not instantiate the adapter')
|
|
|
|
meta = BenchmarkMeta(
|
|
name='runtime_only',
|
|
dataset_id='local',
|
|
metric_list=['accuracy'],
|
|
data_adapter=RuntimeOnlyAdapter,
|
|
)
|
|
|
|
extracted = extract_benchmark_meta(meta, RuntimeOnlyAdapter)
|
|
|
|
assert extracted['metrics'] == ['accuracy']
|
|
assert 'primary_metric' not in extracted
|
|
assert extracted['category'] == 'llm'
|
|
|
|
|
|
def test_doc_metadata_serializes_structured_primary_metric() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='structured_primary',
|
|
dataset_id='local',
|
|
metric_list=['accuracy'],
|
|
primary_metric=MetricSelector(name='accuracy', aggregation='pass_at_k', dimensions={'k': 1}),
|
|
)
|
|
|
|
extracted = extract_benchmark_meta(meta, None)
|
|
|
|
assert extracted['primary_metric'] == {
|
|
'name': 'accuracy',
|
|
'aggregation': 'pass_at_k',
|
|
'dimensions': {
|
|
'k': 1
|
|
},
|
|
}
|
|
json.dumps(extracted)
|