13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
217 lines
9.4 KiB
Python
217 lines
9.4 KiB
Python
"""End-to-end consistency of metric semantics across every output surface.
|
|
|
|
A report is generated once per scenario and then read back through each surface that presents it:
|
|
the CLI DataFrame, the HTML report, the reports API and the report model itself. The point is not
|
|
that each surface works, but that they agree: the same metric must carry the same name, the same
|
|
direction and the same formatted value everywhere, because they all read one contract.
|
|
|
|
No model is called: reports are built from aggregated scores directly, which is what the report
|
|
generator consumes.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
import pytest
|
|
|
|
from evalscope.api.metric import AggScore
|
|
from evalscope.api.metric.semantics import MetricKind, MetricSelector
|
|
from evalscope.metrics.semantics import format_metric_value
|
|
from evalscope.report.report import Report
|
|
from evalscope.utils.data_utils import get_acc_report_df
|
|
|
|
|
|
class _StubAdapter:
|
|
"""Minimal adapter surface the report generator reads."""
|
|
|
|
def __init__(self, name: str, primary_metric: Optional[MetricSelector] = None) -> None:
|
|
self.name = name
|
|
self.primary_metric = primary_metric
|
|
self.pretty_name = name
|
|
self.description = ''
|
|
self.category_map: Dict[str, List[str]] = {}
|
|
|
|
|
|
#: (scenario, benchmark, primary raw name, aggregated scores, expected semantic id of the primary)
|
|
SCENARIOS = [
|
|
(
|
|
'accuracy',
|
|
'gsm8k',
|
|
None,
|
|
[AggScore(score=0.8567, metric_name='accuracy', aggregation='mean', num=100)],
|
|
'quality.accuracy.ratio',
|
|
),
|
|
(
|
|
'wer',
|
|
'torgo',
|
|
MetricSelector(name='wer', aggregation='identity'),
|
|
[
|
|
AggScore(score=0.0432, metric_name='wer', aggregation='identity', num=50),
|
|
AggScore(score=0.0321, metric_name='cer', aggregation='identity', num=50),
|
|
],
|
|
'quality.wer.ratio',
|
|
),
|
|
(
|
|
'pass_at_1',
|
|
'humaneval',
|
|
MetricSelector(name='accuracy', aggregation='pass_at_k', dimensions={'k': 1}),
|
|
[AggScore(score=0.75, metric_name='accuracy', aggregation='pass_at_k', dimensions={'k': 1}, num=164)],
|
|
'quality.pass_at_k.ratio',
|
|
),
|
|
(
|
|
'ner_f1',
|
|
'conll2003',
|
|
MetricSelector(name='f1', aggregation='identity'),
|
|
[
|
|
AggScore(score=0.91, metric_name='f1', aggregation='identity', num=200),
|
|
AggScore(score=0.89, metric_name='precision', aggregation='identity', num=200),
|
|
AggScore(score=0.93, metric_name='recall', aggregation='identity', num=200),
|
|
],
|
|
'quality.f1.ratio',
|
|
),
|
|
(
|
|
'official_points_100',
|
|
'arena_hard',
|
|
None,
|
|
[AggScore(score=87.25, metric_name='weighted_score_percent', aggregation='identity', num=500)],
|
|
'quality.score.points_100',
|
|
),
|
|
(
|
|
'agent_diagnostics',
|
|
'miniwob',
|
|
MetricSelector(name='success_rate', aggregation='mean'),
|
|
[
|
|
AggScore(score=0.62, metric_name='success_rate', aggregation='mean', num=80),
|
|
AggScore(score=0.11, metric_name='error_rate', aggregation='mean', num=80),
|
|
],
|
|
'quality.accuracy.ratio',
|
|
),
|
|
(
|
|
'rubric_dimensions',
|
|
'plawbench',
|
|
MetricSelector(name='accuracy', aggregation='mean'),
|
|
[
|
|
AggScore(score=0.72, metric_name='accuracy', aggregation='mean', num=30),
|
|
AggScore(score=0.68, metric_name='conclusion_acc', aggregation='mean', num=30),
|
|
AggScore(score=0.75, metric_name='fact_acc', aggregation='mean', num=30),
|
|
AggScore(score=0.70, metric_name='reasoning_acc', aggregation='mean', num=30),
|
|
AggScore(score=0.66, metric_name='law_acc', aggregation='mean', num=30),
|
|
],
|
|
'quality.accuracy.ratio',
|
|
),
|
|
]
|
|
|
|
|
|
def _build_report(benchmark: str, primary_metric: Optional[MetricSelector], agg_scores: List[AggScore]) -> Report:
|
|
"""Generate a report the way the evaluator does, then round-trip it through JSON."""
|
|
from evalscope.report.generator import ReportGenerator
|
|
|
|
report = ReportGenerator.generate_report(
|
|
score_dict={'default': agg_scores},
|
|
model_name='test-model',
|
|
data_adapter=_StubAdapter(benchmark, primary_metric=primary_metric),
|
|
)
|
|
# Reading a report back is the path every surface takes, so assert on that shape.
|
|
return Report.from_dict(json.loads(json.dumps(report.to_dict())))
|
|
|
|
|
|
@pytest.mark.parametrize('scenario,benchmark,primary_metric,agg_scores,expected_semantic_id', SCENARIOS)
|
|
class TestSemanticsEndToEnd:
|
|
"""Every metric display surface agrees on the resolved semantics."""
|
|
|
|
@pytest.fixture
|
|
def report(self, benchmark, primary_metric, agg_scores) -> Report:
|
|
"""The scenario's report, built once per test instead of once per assertion."""
|
|
return _build_report(benchmark, primary_metric, agg_scores)
|
|
|
|
def test_primary_metric_resolves_to_the_expected_semantics(
|
|
self, report: Report, scenario, benchmark, primary_metric, agg_scores, expected_semantic_id
|
|
) -> None:
|
|
assert report.primary_metric is not None, scenario
|
|
assert report.primary_metric.semantics.semantic_id == expected_semantic_id
|
|
assert report.primary_metric.semantics.kind is MetricKind.QUALITY
|
|
assert report.primary_metric.identity == report.primary_metric_identity
|
|
|
|
def test_cli_dataframe_matches_the_primary_metric(
|
|
self, report: Report, scenario, benchmark, primary_metric, agg_scores, expected_semantic_id
|
|
) -> None:
|
|
df = get_acc_report_df([report])
|
|
|
|
assert len(df) == 1
|
|
assert df.iloc[0]['Score'] == pytest.approx(report.primary_metric.score)
|
|
assert isinstance(df.iloc[0]['Score'], float)
|
|
|
|
def test_html_report_shows_the_same_formatted_value(
|
|
self, report: Report, scenario, benchmark, primary_metric, agg_scores, expected_semantic_id, tmp_path: Path
|
|
) -> None:
|
|
from evalscope.report.renderer import gen_html_report_file
|
|
|
|
reports_dir = tmp_path / 'reports' / 'test-model'
|
|
reports_dir.mkdir(parents=True)
|
|
(reports_dir / f'{benchmark}.json').write_text(report.to_json_str(), encoding='utf-8')
|
|
|
|
html_path = gen_html_report_file(str(tmp_path / 'reports'))
|
|
html = Path(html_path).read_text(encoding='utf-8')
|
|
|
|
semantics = report.primary_metric.semantics
|
|
expected_value = format_metric_value(report.primary_metric.score, semantics)
|
|
assert semantics.metric_name in html, scenario
|
|
assert expected_value in html, f'{scenario}: expected {expected_value!r} in the HTML report'
|
|
|
|
def test_api_payload_carries_the_same_contract(
|
|
self, report: Report, scenario, benchmark, primary_metric, agg_scores, expected_semantic_id
|
|
) -> None:
|
|
from evalscope.service.blueprints.reports import _report_to_service_dict
|
|
|
|
payload = _report_to_service_dict(report)
|
|
|
|
assert payload['primary_metric_identity'] == report.primary_metric.identity.model_dump()
|
|
by_identity = {json.dumps(metric['identity'], sort_keys=True): metric for metric in payload['metrics']}
|
|
primary_key = json.dumps(report.primary_metric.identity.model_dump(), sort_keys=True)
|
|
primary_payload = by_identity[primary_key]
|
|
assert primary_payload['semantics']['semantic_id'] == expected_semantic_id
|
|
assert primary_payload['semantics']['kind'] == 'quality'
|
|
assert set(('name', 'semantic_id')).isdisjoint(primary_payload)
|
|
assert set(('score', 'primary_metric_name')).isdisjoint(payload)
|
|
|
|
|
|
class TestDirectionsSurviveTheRoundTrip:
|
|
"""A low-is-better metric must not be presented as if higher were better.
|
|
|
|
Asserted on the report read back from JSON, which is the shape every surface consumes; the
|
|
per-surface agreement itself is covered by ``TestSemanticsEndToEnd``. The perf key spaces have
|
|
their own resolution tests in ``tests/report/semantics/test_perf_semantics.py``.
|
|
"""
|
|
|
|
def test_wer_stays_lower_is_better_after_a_round_trip(self) -> None:
|
|
report = _build_report(
|
|
'torgo', MetricSelector(name='wer'), [
|
|
AggScore(score=0.0432, metric_name='wer', aggregation='identity', num=50),
|
|
AggScore(score=0.0321, metric_name='cer', aggregation='identity', num=50),
|
|
]
|
|
)
|
|
|
|
assert report.primary_metric.semantics.direction.value == 'lower_is_better'
|
|
# The supporting error rate stays comparable but is not the conclusion.
|
|
cer = next(metric for metric in report.metrics if metric.identity.name == 'cer')
|
|
assert cer.semantics.kind is MetricKind.QUALITY
|
|
assert cer.semantics.direction.value == 'lower_is_better'
|
|
|
|
def test_diagnostics_never_become_the_conclusion(self) -> None:
|
|
report = _build_report(
|
|
'miniwob',
|
|
MetricSelector(name='success_rate'),
|
|
[
|
|
AggScore(score=0.62, metric_name='success_rate', aggregation='mean', num=80),
|
|
# Emitted without an aggregation prefix, as the adapters that report counts do.
|
|
AggScore(score=0.5, metric_name='no_answer_num', aggregation='identity', num=80),
|
|
]
|
|
)
|
|
|
|
diagnostic = next(metric for metric in report.metrics if metric.identity.name == 'no_answer_num')
|
|
assert diagnostic.semantics.kind is MetricKind.DIAGNOSTIC
|
|
assert diagnostic.semantics.direction.value == 'none'
|
|
assert report.primary_metric.identity.name == 'success_rate'
|
|
assert report.primary_metric.identity.aggregation == 'mean'
|