13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
120 lines
4.4 KiB
Python
120 lines
4.4 KiB
Python
"""Unit tests for the VLMs Are Biased adapter and official scoring rules."""
|
|
import io
|
|
from types import SimpleNamespace
|
|
from typing import Any, Dict
|
|
|
|
from PIL import Image
|
|
|
|
from evalscope.api.metric import MetricDirection, SampleScore
|
|
from evalscope.api.metric.scorer import Score
|
|
from evalscope.api.registry import get_benchmark
|
|
from evalscope.benchmarks.vlms_are_biased.utils import normalize_answer, score_answer
|
|
from evalscope.config import TaskConfig
|
|
from evalscope.metrics.semantics.catalog import METRIC_DEFINITIONS
|
|
|
|
|
|
def _adapter() -> Any:
|
|
task_config = TaskConfig(model='mock', datasets=['vlms_are_biased'])
|
|
return get_benchmark('vlms_are_biased', task_config)
|
|
|
|
|
|
def _image_bytes() -> bytes:
|
|
buffer = io.BytesIO()
|
|
Image.new('RGB', (2, 2)).save(buffer, format='PNG')
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _record(**overrides: Any) -> Dict[str, Any]:
|
|
record = {
|
|
'image': {
|
|
'bytes': _image_bytes(),
|
|
'path': 'sample.png'
|
|
},
|
|
'ID': 'sample_Q1_px384',
|
|
'topic': 'Logos',
|
|
'sub_topic': 'Shoe Logos',
|
|
'prompt': 'How many stripes are visible? Answer with a number in curly brackets, e.g., {9}.',
|
|
'ground_truth': '4',
|
|
'expected_bias': '3',
|
|
'with_title': False,
|
|
'type_of_question': 'Q1',
|
|
'pixel': 384,
|
|
}
|
|
record.update(overrides)
|
|
return record
|
|
|
|
|
|
def test_normalize_answer_matches_official_brace_rule() -> None:
|
|
assert normalize_answer(' {Yes}\n') == 'yes'
|
|
assert normalize_answer('{{4}}') == '4'
|
|
assert normalize_answer('The answer is {4}.') == 'the answer is {4}.'
|
|
|
|
|
|
def test_score_answer_exact_and_bias_matches() -> None:
|
|
assert score_answer('{4}', '4', '3') == {'acc': 1.0, 'bias_ratio': 0.0}
|
|
assert score_answer('{3}', '4', '3') == {'acc': 0.0, 'bias_ratio': 1.0}
|
|
assert score_answer('{2}', '4', '3') == {'acc': 0.0, 'bias_ratio': 0.0}
|
|
assert score_answer('{YES}', 'Yes', 'No') == {'acc': 1.0, 'bias_ratio': 0.0}
|
|
|
|
|
|
def test_score_answer_uses_official_digit_fallback() -> None:
|
|
assert score_answer('There are {4} visible stripes.', '4', '3') == {'acc': 1.0, 'bias_ratio': 0.0}
|
|
assert score_answer('I see 3 stripes.', '4', '3') == {'acc': 0.0, 'bias_ratio': 1.0}
|
|
assert score_answer('I counted 3, then 4.', '4', '3') == {'acc': 0.0, 'bias_ratio': 0.0}
|
|
|
|
|
|
def test_original_control_omits_bias_ratio() -> None:
|
|
assert score_answer('{Audi}', 'Audi', None) == {'acc': 1.0}
|
|
|
|
|
|
def test_bias_ratio_semantics_are_lower_is_better() -> None:
|
|
semantics = METRIC_DEFINITIONS['bias_ratio'].resolve('bias_ratio')
|
|
|
|
assert semantics.metric_name == 'Bias Ratio'
|
|
assert semantics.direction is MetricDirection.LOWER_IS_BETTER
|
|
|
|
|
|
def test_record_to_sample_preserves_official_prompt_and_metadata() -> None:
|
|
sample = _adapter().record_to_sample(_record())
|
|
|
|
assert sample.target == '4'
|
|
assert sample.input[0].content[0].image.startswith('data:image/png;base64,')
|
|
assert sample.input[0].content[1].text == _record()['prompt']
|
|
assert sample.metadata == {
|
|
'id': 'sample_Q1_px384',
|
|
'topic': 'Logos',
|
|
'sub_topic': 'Shoe Logos',
|
|
'type_of_question': 'Q1',
|
|
'expected_bias': '3',
|
|
'with_title': False,
|
|
'pixel': 384,
|
|
}
|
|
|
|
|
|
def test_match_score_uses_expected_bias_metadata() -> None:
|
|
score = _adapter().match_score(
|
|
original_prediction='{3}',
|
|
filtered_prediction='3',
|
|
reference='4',
|
|
task_state=SimpleNamespace(metadata={'expected_bias': '3'}),
|
|
)
|
|
|
|
assert score.value == {'acc': 0.0, 'bias_ratio': 1.0}
|
|
assert score.main_score_name == 'acc'
|
|
|
|
|
|
def test_aggregation_reports_official_per_topic_accuracy() -> None:
|
|
sample_scores = [
|
|
SampleScore(sample_id='1', score=Score(value={'acc': 1.0, 'bias_ratio': 0.0}), sample_metadata={'topic': 'Logos'}),
|
|
SampleScore(sample_id='2', score=Score(value={'acc': 0.0, 'bias_ratio': 1.0}), sample_metadata={'topic': 'Logos'}),
|
|
SampleScore(sample_id='3', score=Score(value={'acc': 1.0, 'bias_ratio': 0.0}), sample_metadata={'topic': 'Flags'}),
|
|
]
|
|
|
|
aggregate_scores = _adapter().aggregate_scores(sample_scores)
|
|
by_identity = {(score.metric_name, score.dimensions.get('topic')): score.score for score in aggregate_scores}
|
|
|
|
assert by_identity[('accuracy', None)] == 2 / 3
|
|
assert by_identity[('bias_ratio', None)] == 1 / 3
|
|
assert by_identity[('accuracy', 'Logos')] == 0.5
|
|
assert by_identity[('accuracy', 'Flags')] == 1.0
|