import json import wave from io import BytesIO from pathlib import Path import pyarrow as pa import pyarrow.parquet as pq import pytest from datasets import Audio, Dataset, Features, Image, Sequence, Value from PIL import Image as PILImage from evalscope.api.benchmark import BenchmarkMeta from evalscope.api.dataset import LocalDataLoader from evalscope.api.messages import ChatMessageUser, ContentAudio, ContentImage, ContentText from evalscope.benchmarks.general_vmcq.general_vmcq_adapter import GeneralVMCQAdapter from evalscope.config import TaskConfig from evalscope.utils.multi_choices import MultipleChoiceTemplate @pytest.fixture def adapter() -> GeneralVMCQAdapter: return GeneralVMCQAdapter( benchmark_meta=BenchmarkMeta( name='general_vmcq', dataset_id='dummy', eval_split='test', prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT, ), task_config=TaskConfig(datasets=['general_vmcq']), ) @pytest.fixture def png_bytes() -> bytes: image = PILImage.new(mode='RGB', size=(10, 10), color=(255, 0, 0)) buffer = BytesIO() image.save(buffer, format='PNG') return buffer.getvalue() @pytest.fixture def jpeg_bytes() -> bytes: image = PILImage.new(mode='RGB', size=(10, 10), color=(0, 0, 255)) buffer = BytesIO() image.save(buffer, format='JPEG') return buffer.getvalue() @pytest.fixture def wav_bytes() -> bytes: buffer = BytesIO() with wave.open(buffer, 'wb') as writer: writer.setnchannels(1) writer.setsampwidth(2) writer.setframerate(8000) writer.writeframes(b'\x00\x00' * 800) return buffer.getvalue() def test_record_to_sample_builds_realistic_image_question(adapter: GeneralVMCQAdapter, tmp_path: Path) -> None: image_path = tmp_path / 'traffic_stop.jpg' sample = adapter.record_to_sample( { 'id': 'traffic-sign-1', 'question': ' Which traffic sign is shown?', 'options': ['Stop', 'Yield', 'No parking', 'School zone'], 'image_1': str(image_path), 'answer': 'A', } ) assert isinstance(sample.input[0], ChatMessageUser) content_list = sample.input[0].content joined_text = ''.join(content.text for content in content_list if isinstance(content, ContentText)) assert sample.target == 'A' assert sample.choices == ['Stop', 'Yield', 'No parking', 'School zone'] assert any(isinstance(content, ContentImage) and content.image == str(image_path) for content in content_list) assert 'Which traffic sign is shown?' in joined_text assert 'Stop' in joined_text def test_record_to_sample_supports_json_options_and_audio_placeholder(adapter: GeneralVMCQAdapter) -> None: sample = adapter.record_to_sample( { 'question': '