13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
191 lines
7.5 KiB
Python
191 lines
7.5 KiB
Python
import plotly.graph_objects as go
|
|
import pytest
|
|
|
|
from evalscope.api.metric.semantics import MetricIdentity
|
|
from evalscope.metrics.semantics.baselines import SEMANTIC_BASELINES
|
|
from evalscope.report import Category, Metric, Report, ReportRef, Subset
|
|
from evalscope.report.visualization import plot_multi_report_radar, plot_single_report_scores
|
|
from evalscope.service.blueprints.reports import _apply_chart_theme, _build_report_meta, _report_to_service_dict
|
|
from evalscope.utils.data_utils import get_comparison_quality_report_df, get_quality_report_df
|
|
|
|
|
|
def test_apply_chart_theme_uses_light_template_for_light_console() -> None:
|
|
fig = go.Figure()
|
|
|
|
_apply_chart_theme(fig, 'light')
|
|
|
|
assert fig.layout.template.layout.plot_bgcolor == 'white'
|
|
assert fig.layout.template.layout.paper_bgcolor == 'white'
|
|
|
|
|
|
def test_apply_chart_theme_keeps_dark_template_as_safe_default() -> None:
|
|
fig = go.Figure()
|
|
|
|
_apply_chart_theme(fig, 'invalid')
|
|
|
|
assert fig.layout.template.layout.plot_bgcolor == 'rgb(17,17,17)'
|
|
assert fig.layout.template.layout.paper_bgcolor == 'rgb(17,17,17)'
|
|
|
|
|
|
def _hydrated(report: Report) -> Report:
|
|
"""Round-trip a report through the read path so its metric semantics are resolved.
|
|
|
|
``Report(...)`` leaves ``Metric.semantics`` unset, and the primary metric is chosen from the
|
|
resolved role rather than from metric order. Production only ever reads a report through
|
|
``from_dict``, so a test that constructs one directly would assert against a state that
|
|
cannot occur.
|
|
"""
|
|
return Report.from_dict(report.to_dict())
|
|
|
|
|
|
def test_build_report_meta_exposes_primary_metric_identity(monkeypatch) -> None:
|
|
identity = MetricIdentity(name='output_throughput', aggregation='mean')
|
|
semantics = SEMANTIC_BASELINES['perf.throughput.tokens_per_second']
|
|
report = Report(
|
|
dataset_name='throughput_suite',
|
|
dataset_pretty_name='Throughput Suite',
|
|
model_name='test-model',
|
|
metrics=[
|
|
Metric(
|
|
identity=identity,
|
|
semantics=semantics,
|
|
categories=[Category(name=('default', ), subsets=[Subset(name='main', score=512.0, num=1)])],
|
|
)
|
|
],
|
|
primary_metric_identity=identity,
|
|
)
|
|
monkeypatch.setattr(
|
|
'evalscope.service.blueprints.reports.get_report_list',
|
|
lambda _paths: [report],
|
|
)
|
|
|
|
metadata = _build_report_meta(ReportRef(run_id='run', model_id='test-model'), '/tmp')
|
|
|
|
assert metadata['run_id'] == 'run'
|
|
assert metadata['model_id'] == 'test-model'
|
|
assert metadata['primary_metrics'] == [{
|
|
'dataset_name': 'throughput_suite',
|
|
'dataset_pretty_name': 'Throughput Suite',
|
|
'identity': identity.model_dump(),
|
|
'score': 512.0,
|
|
'semantics': semantics.model_dump(mode='json'),
|
|
}]
|
|
assert metadata['dataset_name'] == 'throughput_suite'
|
|
assert metadata['dataset_pretty_name'] == 'Throughput Suite'
|
|
assert set(('quality_ratio', '_quality_group')).isdisjoint(metadata)
|
|
assert set(('metric_name', 'score', 'dataset_scores')).isdisjoint(metadata)
|
|
|
|
|
|
def test_build_report_meta_picks_the_primary_identity_not_the_first_metric(monkeypatch) -> None:
|
|
"""The declared structured identity, not metric order, selects the conclusion."""
|
|
accuracy_identity = MetricIdentity(name='accuracy', aggregation='mean')
|
|
normalized_identity = MetricIdentity(name='normalized_score', aggregation='mean')
|
|
accuracy_semantics = SEMANTIC_BASELINES['quality.accuracy.ratio']
|
|
normalized_semantics = SEMANTIC_BASELINES['quality.score.ratio']
|
|
report = Report(
|
|
dataset_name='document_suite',
|
|
model_name='test-model',
|
|
metrics=[
|
|
Metric(
|
|
identity=accuracy_identity,
|
|
semantics=accuracy_semantics,
|
|
categories=[Category(name=('default', ), subsets=[Subset(name='main', score=0.1, num=2)])],
|
|
),
|
|
Metric(
|
|
identity=normalized_identity,
|
|
semantics=normalized_semantics,
|
|
categories=[Category(name=('default', ), subsets=[Subset(name='main', score=0.9, num=2)])],
|
|
),
|
|
],
|
|
primary_metric_identity=normalized_identity,
|
|
)
|
|
monkeypatch.setattr(
|
|
'evalscope.service.blueprints.reports.get_report_list',
|
|
lambda _paths: [report],
|
|
)
|
|
|
|
metadata = _build_report_meta(ReportRef(run_id='run', model_id='test-model'), '/tmp')
|
|
|
|
assert report.primary_metric.score == 0.9
|
|
assert metadata['primary_metrics'][0]['identity'] == normalized_identity.model_dump()
|
|
assert metadata['primary_metrics'][0]['score'] == 0.9
|
|
assert set(('quality_ratio', '_quality_group')).isdisjoint(metadata)
|
|
payload = _report_to_service_dict(report)
|
|
assert payload['primary_metric_identity'] == normalized_identity.model_dump()
|
|
assert set(('score', 'metric_name', 'dataset_scores')).isdisjoint(payload)
|
|
|
|
|
|
def test_build_report_meta_does_not_rank_multiple_datasets(monkeypatch) -> None:
|
|
reports = [
|
|
_semantic_report('accuracy', 0.8, 'quality.accuracy.ratio'),
|
|
_semantic_report('f1', 0.6, 'quality.f1.ratio'),
|
|
]
|
|
monkeypatch.setattr(
|
|
'evalscope.service.blueprints.reports.get_report_list',
|
|
lambda _paths: reports,
|
|
)
|
|
|
|
metadata = _build_report_meta(ReportRef(run_id='run', model_id='test-model'), '/tmp')
|
|
|
|
assert len(metadata['primary_metrics']) == 2
|
|
assert set(('quality_ratio', '_quality_group')).isdisjoint(metadata)
|
|
|
|
|
|
def _semantic_report(dataset_name: str, score: float, semantic_id: str) -> Report:
|
|
semantics = SEMANTIC_BASELINES[semantic_id]
|
|
identity = MetricIdentity(name=dataset_name, aggregation='mean')
|
|
return Report(
|
|
dataset_name=dataset_name,
|
|
model_name='test-model',
|
|
metrics=[
|
|
Metric(
|
|
identity=identity,
|
|
semantics=semantics,
|
|
categories=[Category(name=('default', ), subsets=[Subset(name='main', score=score, num=1)])],
|
|
)
|
|
],
|
|
primary_metric_identity=identity,
|
|
)
|
|
|
|
|
|
def test_comparison_chart_uses_quality_ratio_and_keeps_native_labels() -> None:
|
|
quality_df = get_quality_report_df([
|
|
_semantic_report('points', 87.5, 'quality.score.points_100'),
|
|
_semantic_report('error', 0.2, 'quality.error_rate.ratio'),
|
|
])
|
|
|
|
assert quality_df['Score'].tolist() == pytest.approx([0.875, 0.8])
|
|
assert quality_df['Display Score'].tolist() == ['87.5%', '20%']
|
|
|
|
figure = plot_single_report_scores(quality_df)
|
|
assert figure.layout.yaxis.range == (0, 1)
|
|
assert list(figure.data[0].text) == ['87.5%', '20%']
|
|
|
|
|
|
def test_comparison_chart_omits_unbounded_metrics() -> None:
|
|
quality_df = get_quality_report_df([
|
|
_semantic_report('throughput', 512.0, 'perf.throughput.tokens_per_second'),
|
|
])
|
|
|
|
assert quality_df.empty
|
|
assert plot_single_report_scores(quality_df) is None
|
|
|
|
|
|
def test_comparison_chart_keeps_separate_runs_of_the_same_model() -> None:
|
|
first_ref = ReportRef(run_id='20260810_100000', model_id='test-model')
|
|
second_ref = ReportRef(run_id='20260810_110000', model_id='test-model')
|
|
quality_df = get_comparison_quality_report_df([
|
|
(first_ref, [_semantic_report('points', 75.0, 'quality.score.points_100')]),
|
|
(second_ref, [_semantic_report('points', 90.0, 'quality.score.points_100')]),
|
|
])
|
|
|
|
assert quality_df['Model'].tolist() == [
|
|
'test-model (20260810_100000)',
|
|
'test-model (20260810_110000)',
|
|
]
|
|
figure = plot_multi_report_radar(quality_df)
|
|
assert [trace.name for trace in figure.data] == [
|
|
'test-model (20260810_100000)',
|
|
'test-model (20260810_110000)',
|
|
]
|