13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
204 lines
8.0 KiB
Python
204 lines
8.0 KiB
Python
"""Tests for the perf field semantics.
|
|
|
|
* ``TestPerfFieldCoverage`` -- every public perf field key has an entry and every
|
|
key comes from the perf name constants.
|
|
* ``TestPerfDirections`` -- latency is lower_is_better, throughput is higher_is_better.
|
|
* ``TestPerfDiagnostics`` -- counts, cache and failure details carry no direction.
|
|
"""
|
|
|
|
from typing import Dict, FrozenSet
|
|
|
|
import pytest
|
|
|
|
from evalscope.api.metric.semantics import MetricDirection, MetricKind
|
|
from evalscope.metrics.semantics import attach_perf_semantics, format_perf_value, resolve_perf_semantics
|
|
from evalscope.metrics.semantics.perf import PERF_API_ALIASES, PERF_SEMANTICS
|
|
from evalscope.metrics.semantics.resolver import SemanticsResolver
|
|
from evalscope.perf.utils.perf_constants import Metrics, PercentileMetrics
|
|
|
|
|
|
def _constant_values(holder: type) -> FrozenSet[str]:
|
|
"""Return the public string constants declared on a perf constants holder."""
|
|
return frozenset(
|
|
value for name, value in vars(holder).items() if not name.startswith('_') and isinstance(value, str)
|
|
)
|
|
|
|
|
|
PUBLIC_FIELD_KEYS: FrozenSet[str] = _constant_values(Metrics) | _constant_values(PercentileMetrics)
|
|
|
|
#: Fields whose value gets smaller as the system gets better.
|
|
LOWER_IS_BETTER_FIELDS = (
|
|
Metrics.AVERAGE_LATENCY,
|
|
Metrics.AVERAGE_TIME_TO_FIRST_TOKEN,
|
|
Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN,
|
|
Metrics.AVERAGE_INTER_TOKEN_LATENCY,
|
|
Metrics.AVERAGE_FIRST_TURN_TTFT,
|
|
Metrics.AVERAGE_SUBSEQUENT_TURN_TTFT,
|
|
PercentileMetrics.TTFT,
|
|
PercentileMetrics.ITL,
|
|
PercentileMetrics.TPOT,
|
|
PercentileMetrics.LATENCY,
|
|
)
|
|
|
|
#: Fields whose value gets bigger as the system gets better.
|
|
HIGHER_IS_BETTER_FIELDS = (
|
|
Metrics.REQUEST_THROUGHPUT,
|
|
Metrics.OUTPUT_TOKEN_THROUGHPUT,
|
|
Metrics.TOTAL_TOKEN_THROUGHPUT,
|
|
Metrics.INPUT_TOKEN_THROUGHPUT,
|
|
PercentileMetrics.OUTPUT_THROUGHPUT,
|
|
PercentileMetrics.INPUT_THROUGHPUT,
|
|
PercentileMetrics.TOTAL_THROUGHPUT,
|
|
PercentileMetrics.DECODE_THROUGHPUT,
|
|
)
|
|
|
|
#: Fields that describe the run instead of grading it.
|
|
DIAGNOSTIC_FIELDS = (
|
|
Metrics.TOTAL_REQUESTS,
|
|
Metrics.SUCCEED_REQUESTS,
|
|
Metrics.FAILED_REQUESTS,
|
|
Metrics.STREAM_REQUESTS,
|
|
Metrics.NON_STREAM_REQUESTS,
|
|
Metrics.NUMBER_OF_CONCURRENCY,
|
|
Metrics.AVERAGE_CACHED_PERCENT,
|
|
Metrics.AVERAGE_INPUT_TOKENS_PER_REQUEST,
|
|
Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST,
|
|
PercentileMetrics.INPUT_TOKENS,
|
|
PercentileMetrics.OUTPUT_TOKENS,
|
|
)
|
|
|
|
|
|
class TestPerfFieldCoverage:
|
|
"""The perf semantics map covers every public field key and only uses perf name constants."""
|
|
|
|
def test_every_public_field_is_declared(self) -> None:
|
|
missing = sorted(PUBLIC_FIELD_KEYS - set(PERF_SEMANTICS))
|
|
|
|
assert missing == []
|
|
|
|
@pytest.mark.parametrize('field_key', sorted(PUBLIC_FIELD_KEYS))
|
|
def test_every_entry_resolves(self, field_key: str) -> None:
|
|
semantics = PERF_SEMANTICS[field_key].resolve(field_key)
|
|
|
|
assert semantics.metric_name == field_key
|
|
|
|
|
|
class TestPerfDirections:
|
|
"""Latency and throughput must not be confused with each other."""
|
|
|
|
@pytest.mark.parametrize('field_key', LOWER_IS_BETTER_FIELDS)
|
|
def test_latency_is_lower_is_better(self, field_key: str) -> None:
|
|
semantics = PERF_SEMANTICS[field_key].resolve(field_key)
|
|
|
|
assert semantics.direction is MetricDirection.LOWER_IS_BETTER
|
|
|
|
@pytest.mark.parametrize('field_key', HIGHER_IS_BETTER_FIELDS)
|
|
def test_throughput_is_higher_is_better(self, field_key: str) -> None:
|
|
semantics = PERF_SEMANTICS[field_key].resolve(field_key)
|
|
|
|
assert semantics.direction is MetricDirection.HIGHER_IS_BETTER
|
|
|
|
|
|
class TestPerfDiagnostics:
|
|
"""Counts, cache details and failure details never carry a verdict."""
|
|
|
|
@pytest.mark.parametrize('field_key', DIAGNOSTIC_FIELDS)
|
|
def test_field_is_diagnostic(self, field_key: str) -> None:
|
|
semantics = PERF_SEMANTICS[field_key].resolve(field_key)
|
|
|
|
assert semantics.kind is MetricKind.DIAGNOSTIC
|
|
assert semantics.direction is MetricDirection.NONE
|
|
|
|
|
|
class TestResolvePerfField:
|
|
"""The resolver reads the perf table and degrades for anything it does not declare."""
|
|
|
|
def test_public_field_resolves(self) -> None:
|
|
resolved = SemanticsResolver().resolve_perf_field(Metrics.AVERAGE_LATENCY)
|
|
|
|
assert not resolved.degraded
|
|
assert resolved.semantics.direction is MetricDirection.LOWER_IS_BETTER
|
|
|
|
def test_extension_field_degrades(self) -> None:
|
|
resolved = SemanticsResolver().resolve_perf_field('Some Vendor Extension')
|
|
|
|
assert resolved.degraded
|
|
assert resolved.semantics.kind is MetricKind.DIAGNOSTIC
|
|
assert 'PERF_SEMANTICS' in '\n'.join(resolved.audit_messages)
|
|
|
|
|
|
class TestPerfKeySpaces:
|
|
"""Perf numbers resolve through stable constants and API paths.
|
|
|
|
A response declares semantics under the identifier it exposes the value by. Getting this wrong
|
|
is silent: the consumer looks a key up, misses, and the metric loses its direction and unit
|
|
without any error.
|
|
"""
|
|
|
|
def test_stable_key_spaces_share_one_registry(self) -> None:
|
|
for field_key in (Metrics.AVERAGE_LATENCY, 'best_rps'):
|
|
assert field_key in PERF_SEMANTICS
|
|
|
|
def test_non_constant_key_spaces_are_declared_as_aliases(self) -> None:
|
|
# Asserted as containment, not as a copy of the table: a new alias is a legitimate edit, while
|
|
# an alias pointing at nothing declared is the silent failure worth catching.
|
|
assert set(PERF_API_ALIASES) <= set(PERF_SEMANTICS)
|
|
|
|
@pytest.mark.parametrize(
|
|
'field_key,expected_semantic_id',
|
|
[
|
|
# Perf constants, used by the percentile and summary JSON.
|
|
(Metrics.AVERAGE_LATENCY, 'perf.latency.seconds'),
|
|
# Stable API paths, used by in-report perf and the run list.
|
|
('best_rps', 'perf.throughput.requests_per_second'),
|
|
],
|
|
)
|
|
def test_every_key_space_resolves(self, field_key: str, expected_semantic_id: str) -> None:
|
|
resolved = resolve_perf_semantics([field_key])
|
|
|
|
assert field_key in resolved, f'{field_key!r} did not resolve'
|
|
assert resolved[field_key]['semantic_id'] == expected_semantic_id
|
|
|
|
@pytest.mark.parametrize('field_key', ['success_rate'])
|
|
def test_success_rate_is_already_a_percentage(self, field_key: str) -> None:
|
|
# The perf pipeline formats it as `87.5%`, so scaling it again would render `8750%`.
|
|
semantics = resolve_perf_semantics([field_key])[field_key]
|
|
|
|
assert semantics['display_multiplier'] == 1.0
|
|
assert semantics['display_unit'] == '%'
|
|
assert semantics['value_range'] == {'min': 0.0, 'max': 100.0}
|
|
|
|
@pytest.mark.parametrize('field_key', ['ttft', 'tpot'])
|
|
def test_in_report_streaming_latency_converts_seconds_to_milliseconds(self, field_key: str) -> None:
|
|
semantics = resolve_perf_semantics([field_key])[field_key]
|
|
|
|
assert semantics['raw_unit'] == 's'
|
|
assert semantics['display_multiplier'] == 1000.0
|
|
assert semantics['display_unit'] == 'ms'
|
|
|
|
def test_perf_formatter_uses_registry_precision_and_can_omit_repeated_units(self) -> None:
|
|
assert format_perf_value(1.23456, Metrics.AVERAGE_LATENCY) == '1.235 s'
|
|
assert format_perf_value(1.23456, Metrics.AVERAGE_LATENCY, include_unit=False) == '1.235'
|
|
assert format_perf_value(0.7, Metrics.APPROX_SPECULATIVE_ACCEPTANCE_RATE) == '70%'
|
|
|
|
def test_report_payload_persists_the_semantics_it_displays(self) -> None:
|
|
payload = attach_perf_semantics({
|
|
'summary': {
|
|
'n_samples': 2,
|
|
'latency': {},
|
|
'throughput': {
|
|
'avg_output_tps': 12.5,
|
|
'avg_req_ps': 1.5,
|
|
},
|
|
'usage': {
|
|
'input_tokens': {},
|
|
'output_tokens': {},
|
|
'total_tokens': {},
|
|
},
|
|
'ttft': {},
|
|
}
|
|
})
|
|
|
|
assert payload['metric_semantics']['ttft']['display_unit'] == 'ms'
|
|
assert payload['metric_semantics']['throughput.avg_output_tps']['display_unit'] == 'tok/s'
|