13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
62 lines
2.2 KiB
Python
62 lines
2.2 KiB
Python
"""``llm_recall`` recovers rule-based misses, so the judge can only raise the score.
|
|
|
|
A judge that failed must leave the rule score intact: erasing rule evidence was the defect that
|
|
came from scoring ``[ERROR]`` responses as 0.
|
|
"""
|
|
import pytest
|
|
|
|
from evalscope.api.metric import JudgeSummary, Score
|
|
from evalscope.api.mixin import LLMJudgeMixin
|
|
from evalscope.constants import ScoreStatus
|
|
|
|
|
|
class Merger(LLMJudgeMixin):
|
|
"""Exercises ``_merge_scores`` without needing a benchmark or a judge model."""
|
|
|
|
def __init__(self) -> None:
|
|
pass
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'rule_value, judge_value, expected',
|
|
[
|
|
(0.0, 1.0, 1.0), # the judge recovers a rule miss
|
|
(0.0, 0.0, 0.0),
|
|
(0.6, 0.2, 0.6), # a continuous rule score is not lowered by the judge
|
|
(0.4, 0.9, 0.9),
|
|
],
|
|
)
|
|
def test_merge_takes_the_maximum(rule_value, judge_value, expected):
|
|
rule = Score(value={'acc': rule_value}, main_score_name='acc')
|
|
judge = Score(value={'acc': judge_value}, main_score_name='acc')
|
|
|
|
merged = Merger()._merge_scores(rule, judge)
|
|
|
|
assert merged.value['acc'] == expected
|
|
assert merged.status is ScoreStatus.SUCCESS
|
|
|
|
|
|
@pytest.mark.parametrize('status', [ScoreStatus.TRANSPORT_ERROR, ScoreStatus.PARSE_ERROR, ScoreStatus.EXCLUDED])
|
|
def test_an_unusable_judge_keeps_the_rule_score(status):
|
|
rule = Score(value={'acc': 0.6}, main_score_name='acc')
|
|
judge = Score(value={}, status=status, judge_summary=JudgeSummary(judge_models=['j']))
|
|
|
|
merged = Merger()._merge_scores(rule, judge)
|
|
|
|
assert merged.value == {'acc': 0.6}
|
|
assert merged.status is ScoreStatus.FALLBACK
|
|
assert merged.status.is_usable, 'a rule-scored sample must still count towards the metric'
|
|
assert merged.metadata['judge_unavailable'] == status.value
|
|
assert merged.judge_summary.judge_models == ['j']
|
|
assert merged.judge_summary.status is ScoreStatus.FALLBACK
|
|
|
|
|
|
def test_a_judge_fallback_score_is_still_merged():
|
|
rule = Score(value={'acc': 0.0}, main_score_name='acc')
|
|
judge = Score(value={'acc': 1.0}, status=ScoreStatus.FALLBACK, main_score_name='acc')
|
|
|
|
merged = Merger()._merge_scores(rule, judge)
|
|
|
|
assert merged.value['acc'] == 1.0
|
|
assert merged.status is ScoreStatus.FALLBACK
|