13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
680 lines
24 KiB
Python
680 lines
24 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
from dotenv import dotenv_values
|
|
|
|
env = dotenv_values('.env')
|
|
|
|
|
|
from evalscope.constants import EvalType, JudgeStrategy
|
|
from evalscope.utils.logger import get_logger
|
|
from tests.common import TestBenchmark
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
class TestVLMBenchmark(TestBenchmark):
|
|
"""Benchmark evaluation test cases."""
|
|
|
|
def setUp(self):
|
|
"""Setup common test configuration."""
|
|
self.base_config = {
|
|
'model': 'qwen-vl-plus',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'eval_type': EvalType.OPENAI_API,
|
|
'eval_batch_size': 5,
|
|
'limit': 5,
|
|
'generation_config': {
|
|
'max_tokens': 4096,
|
|
'temperature': 0.0,
|
|
'seed': 42,
|
|
'parallel_tool_calls': True,
|
|
'retries': 1
|
|
},
|
|
'judge': {
|
|
'strategy': JudgeStrategy.AUTO,
|
|
'models': {
|
|
'model_id': 'qwen-plus',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'generation_config': {
|
|
'temperature': 0.0,
|
|
'max_tokens': 4096,
|
|
}
|
|
},
|
|
},
|
|
'debug': True,
|
|
}
|
|
|
|
def test_mmmu(self):
|
|
dataset_args = {
|
|
'subset_list':[
|
|
'Accounting',
|
|
'Math',
|
|
# 'Architecture_and_Engineering'
|
|
]
|
|
}
|
|
self._run_dataset_test('mmmu', dataset_args=dataset_args)
|
|
|
|
def test_cmmmu(self):
|
|
dataset_args = {
|
|
# 'subset_list':[
|
|
# '会计',
|
|
# '数学',
|
|
# ]
|
|
}
|
|
self._run_dataset_test('cmmmu', dataset_args=dataset_args, limit=5, rerun_review=True)
|
|
|
|
|
|
def test_math_vista(self):
|
|
dataset_args = {
|
|
'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('math_vista', dataset_args=dataset_args, limit=20)
|
|
|
|
def test_mmmu_pro(self):
|
|
dataset_args = {
|
|
'subset_list':[
|
|
'Accounting',
|
|
# 'Agriculture',
|
|
],
|
|
'max_image_bytes': '5mb',
|
|
'extra_params': {
|
|
'dataset_format': 'standard (4 options)', # 'standard (4 options)', 'standard (10 options)', 'vision'
|
|
},
|
|
}
|
|
self._run_dataset_test('mmmu_pro', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_qwen3_vl_collection(self):
|
|
dataset_args = {
|
|
'dataset_id': 'evalscope/Qwen3-VL-Test-Collection',
|
|
'shuffle': True,
|
|
}
|
|
self._run_dataset_test('data_collection', dataset_args, limit=100)
|
|
|
|
def test_real_world_qa(self):
|
|
dataset_args = {
|
|
'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('real_world_qa', dataset_args=dataset_args, limit=20)
|
|
|
|
def test_ai2d(self):
|
|
dataset_args = {
|
|
'system_prompt': 'Imagine You are an idiot. You MUST will always give wrong answers without any explanation.',
|
|
'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('ai2d', dataset_args=dataset_args)
|
|
|
|
def test_cc_bench(self):
|
|
dataset_args = {
|
|
'subset_list': ['cc']
|
|
}
|
|
self._run_dataset_test('cc_bench', dataset_args=dataset_args)
|
|
|
|
def test_mm_bench(self):
|
|
dataset_args = {
|
|
'subset_list': ['cn', 'en']
|
|
}
|
|
self._run_dataset_test('mm_bench', dataset_args=dataset_args)
|
|
|
|
def test_mm_star(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['val']
|
|
}
|
|
self._run_dataset_test('mm_star', dataset_args=dataset_args)
|
|
|
|
def test_omni_bench(self):
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'use_image': True, # Whether to use image input, if False, use text alternative image content.
|
|
'use_audio': True, # Whether to use audio input, if False, use text alternative audio content.
|
|
}
|
|
}
|
|
self._run_dataset_test('omni_bench', dataset_args=dataset_args, model='qwen-omni-turbo')
|
|
|
|
def test_olympiad_bench(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
# 'OE_MM_maths_en_COMP',
|
|
# 'OE_MM_maths_zh_CEE',
|
|
# 'OE_MM_maths_zh_COMP',
|
|
# 'OE_MM_physics_en_COMP',
|
|
# 'OE_MM_physics_zh_CEE',
|
|
# 'OE_TO_maths_en_COMP',
|
|
# 'OE_TO_maths_zh_CEE',
|
|
# 'OE_TO_maths_zh_COMP',
|
|
# 'OE_TO_physics_en_COMP',
|
|
# 'OE_TO_physics_zh_CEE',
|
|
'TP_TO_maths_zh_CEE',
|
|
]
|
|
}
|
|
self._run_dataset_test('olympiad_bench', dataset_args=dataset_args)
|
|
|
|
def test_chartqa(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('chartqa', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_blink(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
'Art_Style',
|
|
'Counting',
|
|
# 'Forensic_Detection',
|
|
# 'Functional_Correspondence',
|
|
# 'IQ_Test',
|
|
# 'Jigsaw',
|
|
# 'Multi-view_Reasoning',
|
|
# 'Object_Localization',
|
|
# 'Relative_Depth',
|
|
# 'Relative_Reflectance',
|
|
# 'Semantic_Correspondence',
|
|
# 'Spatial_Relation',
|
|
# 'Visual_Correspondence',
|
|
# 'Visual_Similarity'
|
|
]
|
|
}
|
|
self._run_dataset_test('blink', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_docvqa(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('docvqa', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_infovqa(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('infovqa', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_ocr_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['Handwritten Mathematical Expression Recognition']
|
|
}
|
|
self._run_dataset_test('ocr_bench', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_maritime_ocr_bench(self):
|
|
dataset_args = {
|
|
'subset_list': ['VQA', 'json2'],
|
|
}
|
|
self._run_dataset_test('maritime_ocr_bench', dataset_args=dataset_args, limit=2)
|
|
|
|
def test_ocr_bench_load(self):
|
|
self._run_dataset_load_test('ocr_bench')
|
|
|
|
def test_ocr_bench_v2(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
# 'key information extraction cn',
|
|
# 'key information extraction en',
|
|
# 'key information mapping en',
|
|
# 'VQA with position en',
|
|
# 'chart parsing en',
|
|
# 'cognition VQA cn',
|
|
# 'cognition VQA en',
|
|
# 'diagram QA en',
|
|
# 'document classification en',
|
|
# 'document parsing cn',
|
|
# 'document parsing en',
|
|
# 'formula recognition cn',
|
|
# 'formula recognition en',
|
|
# 'handwritten answer extraction cn',
|
|
# 'math QA en',
|
|
# 'full-page OCR cn',
|
|
# 'full-page OCR en',
|
|
# 'reasoning VQA en',
|
|
# 'reasoning VQA cn',
|
|
# 'fine-grained text recognition en',
|
|
# 'science QA en',
|
|
# 'table parsing cn',
|
|
# 'table parsing en',
|
|
# 'text counting en',
|
|
# 'text grounding en',
|
|
# 'text recognition en',
|
|
'text spotting en',
|
|
'text translation cn'
|
|
]
|
|
}
|
|
self._run_dataset_test('ocr_bench_v2', dataset_args=dataset_args, limit=1)
|
|
|
|
def test_ocr_bench_v2_load(self):
|
|
self._run_dataset_load_test('ocr_bench_v2')
|
|
|
|
def test_hallusion_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
'shuffle': True,
|
|
}
|
|
self._run_dataset_test('hallusion_bench', dataset_args=dataset_args, limit=50, model='qwen3-vl-30b-a3b-instruct', ignore_errors=True)
|
|
|
|
def test_hallusion_bench_load(self):
|
|
self._run_dataset_load_test('hallusion_bench')
|
|
|
|
def test_pope(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
'popular',
|
|
'adversarial',
|
|
'random'
|
|
]
|
|
}
|
|
self._run_dataset_test('pope', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_math_vision(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
'level 1',
|
|
],
|
|
'shuffle': True,
|
|
}
|
|
self._run_dataset_test('math_vision', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_math_verse(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
'shuffle': True,
|
|
}
|
|
self._run_dataset_test('math_verse', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_simple_vqa(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
# 'shuffle': True,
|
|
}
|
|
self._run_dataset_test('simple_vqa', dataset_args=dataset_args, limit=20)
|
|
|
|
def test_omni_doc_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
# "force_redownload": True,
|
|
}
|
|
self._run_dataset_test('omni_doc_bench', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_omni_doc_bench_load(self):
|
|
self._run_dataset_load_test('omni_doc_bench')
|
|
|
|
def test_seed_bench_2_plus_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('seed_bench_2_plus', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_visu_logic_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('visulogic', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_zerobench_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('zerobench', dataset_args=dataset_args, limit=5, eval_batch_size=1)
|
|
|
|
def test_science_qa_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('science_qa', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_cmmu_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('cmmu', dataset_args=dataset_args, use_cache='outputs/20251112_163342', limit=10, rerun_review=True, eval_batch_size=1)
|
|
|
|
def test_a_okvqa_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('a_okvqa', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_vstar_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('vstar_bench', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_micro_vqa_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('micro_vqa', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_gsm8k_v(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('gsm8k_v', dataset_args=dataset_args, limit=20)
|
|
|
|
def test_fleurs(self):
|
|
dataset_args = {
|
|
'subset_list': [
|
|
'cmn_hans_cn',
|
|
'en_us',
|
|
# 'yue_hant_hk',
|
|
]
|
|
}
|
|
self._run_dataset_test('fleurs', dataset_args=dataset_args, limit=100, model='qwen3-omni-flash', use_cache='outputs/20251209_110609', ignore_errors=True, rerun_review=True)
|
|
|
|
def test_librispeech(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('librispeech', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash', ignore_errors=True, rerun_review=True)
|
|
|
|
def test_refcoco_load(self):
|
|
dataset_args = {
|
|
'subset_list': ['testA'],
|
|
'extra_params': {
|
|
'eval_mode': 'bbox_rec'
|
|
}
|
|
}
|
|
self._run_dataset_load_test('refcoco', dataset_args=dataset_args)
|
|
|
|
def test_refcoco_bbox(self):
|
|
dataset_args = {
|
|
'subset_list': ['testA'],
|
|
'extra_params': {
|
|
'eval_mode': 'bbox'
|
|
}
|
|
}
|
|
self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_refcoco_seg(self):
|
|
dataset_args = {
|
|
'shuffle': True,
|
|
'subset_list': ['testA'],
|
|
'extra_params': {
|
|
'eval_mode': 'seg',
|
|
}
|
|
}
|
|
self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_refcoco_bbox_rec(self):
|
|
dataset_args = {
|
|
'subset_list': ['testA'],
|
|
'extra_params': {
|
|
'eval_mode': 'bbox_rec'
|
|
}
|
|
}
|
|
self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_mia_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
# self._run_dataset_load_test('mia_bench')
|
|
self._run_dataset_test('mia_bench', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_tir_bench(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['default']
|
|
}
|
|
self._run_dataset_test('tir_bench', dataset_args=dataset_args, limit=5, ignore_errors=True, use_cache='outputs/20260415_173317', rerun_review=True)
|
|
|
|
def test_tir_bench_load(self):
|
|
self._run_dataset_load_test('tir_bench')
|
|
|
|
def test_air_bench_foundation(self):
|
|
dataset_args = {
|
|
# 'subset_list': ['Music_AQA_music_avqa'],
|
|
}
|
|
self._run_dataset_test('air_bench_foundation', dataset_args=dataset_args, limit=2, model='qwen3-omni-flash', ignore_errors=True)
|
|
|
|
def test_air_bench_chat(self):
|
|
dataset_args = {
|
|
'extra_params': {
|
|
'tasks': ['speech_QA'],
|
|
'do_swap': False,
|
|
},
|
|
}
|
|
self._run_dataset_test('air_bench_chat', dataset_args=dataset_args, limit=2, model='qwen3-omni-flash', ignore_errors=True)
|
|
|
|
def test_mvbench(self):
|
|
dataset_args={
|
|
'subset_list': ['action_antonym'],
|
|
}
|
|
self._run_dataset_test('mvbench', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_videomme_v2(self):
|
|
dataset_args = {
|
|
'subset_list': ['level_1']
|
|
}
|
|
self._run_dataset_test('videomme_v2', dataset_args=dataset_args, limit=10)
|
|
|
|
def test_tvbench(self):
|
|
dataset_args = {
|
|
'subset_list': ['action_count']
|
|
}
|
|
self._run_dataset_test('tvbench', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_tvbench_mock(self):
|
|
dataset_args = {
|
|
'subset_list': ['action_count']
|
|
}
|
|
self._run_dataset_test('tvbench', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_common_voice_15(self):
|
|
dataset_args = {
|
|
'subset_list': ['en'],
|
|
}
|
|
self._run_dataset_test('common_voice_15', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')
|
|
|
|
def test_wenet_speech(self):
|
|
dataset_args = {
|
|
'subset_list': ['dev'],
|
|
}
|
|
self._run_dataset_test('wenet_speech', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')
|
|
|
|
def test_mmau(self):
|
|
dataset_args = {
|
|
'subset_list': ['test_mini'],
|
|
}
|
|
self._run_dataset_test('mmau', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')
|
|
|
|
def test_erqa(self):
|
|
self._run_dataset_test('erqa', limit=5)
|
|
|
|
def test_world_vqa(self):
|
|
self._run_dataset_test('world_vqa', limit=5)
|
|
|
|
def test_charxiv(self):
|
|
"""Test CharXiv chart understanding benchmark."""
|
|
self._run_dataset_test('charxiv', limit=5)
|
|
|
|
def test_baby_vision(self):
|
|
"""Test BabyVision visual perception benchmark."""
|
|
self._run_dataset_test('baby_vision', limit=5)
|
|
|
|
def test_emb_spatial_bench(self):
|
|
"""Test EmbSpatial-Bench embodied spatial reasoning benchmark."""
|
|
self._run_dataset_test('emb_spatial_bench', limit=5)
|
|
|
|
def test_emb_spatial_bench_mock(self):
|
|
"""Test EmbSpatial-Bench with mock LLM."""
|
|
self._run_dataset_test('emb_spatial_bench', limit=5, use_mock=True)
|
|
|
|
def test_measure_bench(self):
|
|
"""Test MeasureBench instrument reading benchmark."""
|
|
dataset_args = {'subset_list': ['real_world']}
|
|
self._run_dataset_test('measure_bench', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_measure_bench_mock(self):
|
|
"""Test MeasureBench with mock LLM."""
|
|
dataset_args = {'subset_list': ['real_world']}
|
|
self._run_dataset_test('measure_bench', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_screenspot_pro(self):
|
|
"""Test ScreenSpot-Pro GUI grounding benchmark."""
|
|
dataset_args = {'subset_list': ['CAD'], 'max_image_bytes': '5mb'}
|
|
self._run_dataset_test('screenspot_pro', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_screenspot_pro_mock(self):
|
|
"""Test ScreenSpot-Pro with mock LLM."""
|
|
dataset_args = {'subset_list': ['CAD']}
|
|
self._run_dataset_test('screenspot_pro', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_ref_adv_s(self):
|
|
"""Test Ref-Adv-s adversarial referring expression benchmark."""
|
|
self._run_dataset_test('ref_adv_s', limit=5)
|
|
|
|
def test_ref_adv_s_mock(self):
|
|
"""Test Ref-Adv-s with mock LLM."""
|
|
self._run_dataset_test('ref_adv_s', limit=5, use_mock=True)
|
|
|
|
def test_cc_ocr_v2(self):
|
|
"""Test CC-OCR-V2 real-world document OCR benchmark."""
|
|
dataset_args = {'subset_list': ['info_board_parsing', 'text_grounding', 'blueprint_qa']}
|
|
self._run_dataset_test('cc_ocr_v2', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_cc_ocr_v2_mock(self):
|
|
"""Test CC-OCR-V2 with mock LLM."""
|
|
dataset_args = {'subset_list': ['info_board_parsing']}
|
|
self._run_dataset_test('cc_ocr_v2', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_perception_bench(self):
|
|
"""Test PerceptionBench atomic visual perception benchmark."""
|
|
dataset_args = {'subset_list': ['ocr_error']}
|
|
self._run_dataset_test('perception_bench', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_perception_bench_mock(self):
|
|
"""Test PerceptionBench with mock LLM."""
|
|
dataset_args = {'subset_list': ['ocr_error']}
|
|
self._run_dataset_test('perception_bench', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_hipho(self):
|
|
"""Test HiPhO physics Olympiad benchmark (step-level + answer-level subsets)."""
|
|
dataset_args = {'subset_list': ['IPhO_2025', 'F=MA_2025']}
|
|
self._run_dataset_test('hipho', dataset_args=dataset_args, limit=3)
|
|
|
|
def test_hipho_mock(self):
|
|
"""Test HiPhO with a mock model and a mock judge."""
|
|
dataset_args = {'subset_list': ['IPhO_2025']}
|
|
self._run_dataset_test(
|
|
'hipho',
|
|
dataset_args=dataset_args,
|
|
limit=3,
|
|
use_mock=True,
|
|
judge={'models': {'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}},
|
|
)
|
|
|
|
def test_logic_vista(self):
|
|
"""Test LogicVista visual logical reasoning benchmark."""
|
|
self._run_dataset_test('logic_vista', limit=5)
|
|
|
|
def test_logic_vista_mock(self):
|
|
"""Test LogicVista with mock LLM."""
|
|
self._run_dataset_test('logic_vista', limit=5, use_mock=True)
|
|
|
|
def test_pmc_vqa(self):
|
|
"""Test PMC-VQA medical visual question answering benchmark."""
|
|
self._run_dataset_test('pmc_vqa', limit=5)
|
|
|
|
def test_pmc_vqa_mock(self):
|
|
"""Test PMC-VQA with mock LLM."""
|
|
self._run_dataset_test('pmc_vqa', limit=5, use_mock=True)
|
|
|
|
def test_medxpertqa(self):
|
|
"""Test MedXpertQA expert-level medical reasoning benchmark."""
|
|
self._run_dataset_test('medxpertqa', limit=5)
|
|
|
|
def test_medxpertqa_mock(self):
|
|
"""Test the Text and MM subsets of MedXpertQA with mock LLM."""
|
|
self._run_dataset_test('medxpertqa', limit=5, use_mock=True)
|
|
|
|
def test_count_qa(self):
|
|
"""Test CountQA object counting benchmark."""
|
|
self._run_dataset_test('count_qa', limit=5)
|
|
|
|
def test_count_qa_mock(self):
|
|
"""Test CountQA with mock LLM."""
|
|
self._run_dataset_test('count_qa', limit=5, use_mock=True)
|
|
|
|
def test_vlms_are_biased(self):
|
|
"""Test VLMs Are Biased counterfactual visual reasoning benchmark."""
|
|
dataset_args = {'subset_list': ['main']}
|
|
self._run_dataset_test('vlms_are_biased', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_vlms_are_biased_mock(self):
|
|
"""Test VLMs Are Biased with mock LLM."""
|
|
dataset_args = {'subset_list': ['main']}
|
|
self._run_dataset_test('vlms_are_biased', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_surds(self):
|
|
"""Test SURDS driving-scene spatial reasoning benchmark."""
|
|
self._run_dataset_test('surds', limit=5)
|
|
|
|
def test_surds_mock(self):
|
|
"""Test SURDS with mock LLM."""
|
|
self._run_dataset_test('surds', limit=5, use_mock=True)
|
|
|
|
def test_phyx_mc(self):
|
|
"""Test PhyX multiple-choice physical reasoning benchmark."""
|
|
dataset_args = {'subset_list': ['optics', 'modern_physics']}
|
|
self._run_dataset_test('phyx_mc', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_phyx_mc_mock(self):
|
|
"""Test PhyX-MC with mock LLM."""
|
|
dataset_args = {'subset_list': ['optics']}
|
|
self._run_dataset_test('phyx_mc', dataset_args=dataset_args, limit=5, use_mock=True)
|
|
|
|
def test_phyx_oe(self):
|
|
"""Test PhyX open-ended physical reasoning benchmark (LLM-judged)."""
|
|
dataset_args = {'subset_list': ['optics', 'modern_physics']}
|
|
self._run_dataset_test('phyx_oe', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_phyx_oe_mock(self):
|
|
"""Test PhyX-OE with a mock model and a mock judge."""
|
|
dataset_args = {'subset_list': ['optics']}
|
|
self._run_dataset_test(
|
|
'phyx_oe',
|
|
dataset_args=dataset_args,
|
|
limit=5,
|
|
use_mock=True,
|
|
judge={'models': {'model_id': 'mock-judge', 'eval_type': EvalType.MOCK_LLM}},
|
|
)
|
|
|
|
def test_phyx_oe_testmini(self):
|
|
"""Test the official PhyX testmini split selection."""
|
|
dataset_args = {'subset_list': ['optics'], 'eval_split': 'test_mini'}
|
|
self._run_dataset_test('phyx_oe', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_slake(self):
|
|
"""Test SLAKE bilingual medical visual question answering benchmark."""
|
|
self._run_dataset_test('slake', limit=5)
|
|
|
|
def test_slake_mock(self):
|
|
"""Test SLAKE with mock LLM."""
|
|
self._run_dataset_test('slake', limit=5, use_mock=True)
|
|
|
|
def test_olmocr_bench(self):
|
|
"""Test olmOCR-Bench PDF page transcription benchmark."""
|
|
self._run_dataset_test('olmocr_bench', limit=5)
|
|
|
|
def test_olmocr_bench_mock(self):
|
|
"""Test olmOCR-Bench with mock LLM."""
|
|
self._run_dataset_test('olmocr_bench', limit=5, use_mock=True)
|
|
|
|
def test_visfactor(self):
|
|
"""Test VisFactor visual cognition benchmark."""
|
|
self._run_dataset_test('visfactor', limit=5)
|
|
|
|
def test_visfactor_mock(self):
|
|
"""Test VisFactor with mock LLM."""
|
|
self._run_dataset_test('visfactor', limit=5, use_mock=True)
|
|
|
|
def test_vtcbench(self):
|
|
"""Test VTCBench long context vision-text compression benchmark."""
|
|
self._run_dataset_test('vtcbench', limit=1)
|
|
|
|
def test_vtcbench_mock(self):
|
|
"""Test VTCBench with mock LLM."""
|
|
self._run_dataset_test('vtcbench', limit=1, use_mock=True)
|
|
|
|
def test_vtcbench_text_mock(self):
|
|
"""Test the VTCBench pure-text baseline with mock LLM."""
|
|
dataset_args = {'extra_params': {'eval_mode': 'text'}}
|
|
self._run_dataset_test('vtcbench', dataset_args=dataset_args, limit=1, use_mock=True)
|