# Copyright (c) Alibaba, Inc. and its affiliates.
from dotenv import dotenv_values

env = dotenv_values('.env')


from evalscope.constants import EvalType, JudgeStrategy
from evalscope.utils.logger import get_logger
from tests.common import TestBenchmark

logger = get_logger()


class TestVLMBenchmark(TestBenchmark):
    """Benchmark evaluation test cases."""

    def setUp(self):
        """Setup common test configuration."""
        self.base_config = {
            'model': 'qwen-vl-plus',
            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
            'api_key': env.get('DASHSCOPE_API_KEY'),
            'eval_type': EvalType.OPENAI_API,
            'eval_batch_size': 5,
            'limit': 5,
            'generation_config': {
                'max_tokens': 4096,
                'temperature': 0.0,
                'seed': 42,
                'parallel_tool_calls': True,
                'retries': 1
            },
            'judge_strategy': JudgeStrategy.AUTO,
            'judge_model_args': {
                'model_id': 'qwen-plus',
                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                'api_key': env.get('DASHSCOPE_API_KEY'),
                'generation_config': {
                    'temperature': 0.0,
                    'max_tokens': 4096,
                }
            },
            'debug': True,
        }

    def test_mmmu(self):
        dataset_args = {
            'subset_list':[
                'Accounting',
                'Math',
                # 'Architecture_and_Engineering'
            ]
        }
        self._run_dataset_test('mmmu', dataset_args=dataset_args)

    def test_cmmmu(self):
        dataset_args = {
            # 'subset_list':[
            #     '会计',
            #     '数学',
            # ]
        }
        self._run_dataset_test('cmmmu', dataset_args=dataset_args, limit=5, rerun_review=True)


    def test_math_vista(self):
        dataset_args = {
            'subset_list': ['default']
        }
        self._run_dataset_test('math_vista', dataset_args=dataset_args, limit=20)

    def test_mmmu_pro(self):
        dataset_args = {
            'subset_list':[
                'Accounting',
                # 'Agriculture',
            ],
            'max_image_bytes': '5mb',
            'extra_params': {
                'dataset_format': 'standard (4 options)',  # 'standard (4 options)', 'standard (10 options)', 'vision'
            },
        }
        self._run_dataset_test('mmmu_pro', dataset_args=dataset_args, limit=10)

    def test_qwen3_vl_collection(self):
        dataset_args = {
            'dataset_id': 'evalscope/Qwen3-VL-Test-Collection',
            'shuffle': True,
        }
        self._run_dataset_test('data_collection', dataset_args, limit=100)

    def test_real_world_qa(self):
        dataset_args = {
            'subset_list': ['default']
        }
        self._run_dataset_test('real_world_qa', dataset_args=dataset_args, limit=20)

    def test_ai2d(self):
        dataset_args = {
            'system_prompt': 'Imagine You are an idiot. You MUST will always give wrong answers without any explanation.',
            'subset_list': ['default']
        }
        self._run_dataset_test('ai2d', dataset_args=dataset_args)

    def test_cc_bench(self):
        dataset_args = {
            'subset_list': ['cc']
        }
        self._run_dataset_test('cc_bench', dataset_args=dataset_args)

    def test_mm_bench(self):
        dataset_args = {
            'subset_list': ['cn', 'en']
        }
        self._run_dataset_test('mm_bench', dataset_args=dataset_args)

    def test_mm_star(self):
        dataset_args = {
            # 'subset_list': ['val']
        }
        self._run_dataset_test('mm_star', dataset_args=dataset_args)

    def test_omni_bench(self):
        dataset_args = {
            'extra_params': {
                'use_image': True, # Whether to use image input, if False, use text alternative image content.
                'use_audio': True, # Whether to use audio input, if False, use text alternative audio content.
            }
        }
        self._run_dataset_test('omni_bench', dataset_args=dataset_args, model='qwen-omni-turbo')

    def test_olympiad_bench(self):
        dataset_args = {
            'subset_list': [
                # 'OE_MM_maths_en_COMP',
                # 'OE_MM_maths_zh_CEE',
                # 'OE_MM_maths_zh_COMP',
                # 'OE_MM_physics_en_COMP',
                # 'OE_MM_physics_zh_CEE',
                # 'OE_TO_maths_en_COMP',
                # 'OE_TO_maths_zh_CEE',
                # 'OE_TO_maths_zh_COMP',
                # 'OE_TO_physics_en_COMP',
                # 'OE_TO_physics_zh_CEE',
                'TP_TO_maths_zh_CEE',
            ]
        }
        self._run_dataset_test('olympiad_bench', dataset_args=dataset_args)

    def test_chartqa(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('chartqa', dataset_args=dataset_args, limit=10)

    def test_blink(self):
        dataset_args = {
            'subset_list': [
                'Art_Style',
                'Counting',
                # 'Forensic_Detection',
                # 'Functional_Correspondence',
                # 'IQ_Test',
                # 'Jigsaw',
                # 'Multi-view_Reasoning',
                # 'Object_Localization',
                # 'Relative_Depth',
                # 'Relative_Reflectance',
                # 'Semantic_Correspondence',
                # 'Spatial_Relation',
                # 'Visual_Correspondence',
                # 'Visual_Similarity'
            ]
        }
        self._run_dataset_test('blink', dataset_args=dataset_args, limit=10)

    def test_docvqa(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('docvqa', dataset_args=dataset_args, limit=5)

    def test_infovqa(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('infovqa', dataset_args=dataset_args, limit=5)

    def test_ocr_bench(self):
        dataset_args = {
            # 'subset_list': ['Handwritten Mathematical Expression Recognition']
        }
        self._run_dataset_test('ocr_bench', dataset_args=dataset_args, limit=10)

    def test_maritime_ocr_bench(self):
        dataset_args = {
            'subset_list': ['VQA', 'json2'],
        }
        self._run_dataset_test('maritime_ocr_bench', dataset_args=dataset_args, limit=2)

    def test_ocr_bench_load(self):
        self._run_dataset_load_test('ocr_bench')

    def test_ocr_bench_v2(self):
        dataset_args = {
            'subset_list': [
                # 'key information extraction cn',
                # 'key information extraction en',
                # 'key information mapping en',
                # 'VQA with position en',
                # 'chart parsing en',
                # 'cognition VQA cn',
                # 'cognition VQA en',
                # 'diagram QA en',
                # 'document classification en',
                # 'document parsing cn',
                # 'document parsing en',
                # 'formula recognition cn',
                # 'formula recognition en',
                # 'handwritten answer extraction cn',
                # 'math QA en',
                # 'full-page OCR cn',
                # 'full-page OCR en',
                # 'reasoning VQA en',
                # 'reasoning VQA cn',
                # 'fine-grained text recognition en',
                # 'science QA en',
                # 'table parsing cn',
                # 'table parsing en',
                # 'text counting en',
                # 'text grounding en',
                # 'text recognition en',
                'text spotting en',
                'text translation cn'
            ]
        }
        self._run_dataset_test('ocr_bench_v2', dataset_args=dataset_args, limit=1)

    def test_ocr_bench_v2_load(self):
        self._run_dataset_load_test('ocr_bench_v2')

    def test_hallusion_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
            'shuffle': True,
        }
        self._run_dataset_test('hallusion_bench', dataset_args=dataset_args, limit=50, model='qwen3-vl-30b-a3b-instruct', ignore_errors=True)

    def test_hallusion_bench_load(self):
        self._run_dataset_load_test('hallusion_bench')

    def test_pope(self):
        dataset_args = {
            'subset_list': [
                'popular',
                'adversarial',
                'random'
            ]
        }
        self._run_dataset_test('pope', dataset_args=dataset_args, limit=5)

    def test_math_vision(self):
        dataset_args = {
            'subset_list': [
                'level 1',
            ],
            'shuffle': True,
        }
        self._run_dataset_test('math_vision', dataset_args=dataset_args, limit=5)

    def test_math_verse(self):
        dataset_args = {
            # 'subset_list': ['default']
            'shuffle': True,
        }
        self._run_dataset_test('math_verse', dataset_args=dataset_args, limit=10)

    def test_simple_vqa(self):
        dataset_args = {
            # 'subset_list': ['default']
            # 'shuffle': True,
        }
        self._run_dataset_test('simple_vqa', dataset_args=dataset_args, limit=20)

    def test_omni_doc_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
            # "force_redownload": True,
        }
        self._run_dataset_test('omni_doc_bench', dataset_args=dataset_args, limit=10)

    def test_omni_doc_bench_load(self):
        self._run_dataset_load_test('omni_doc_bench')

    def test_seed_bench_2_plus_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('seed_bench_2_plus', dataset_args=dataset_args, limit=10)

    def test_visu_logic_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('visulogic', dataset_args=dataset_args, limit=10)

    def test_zerobench_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('zerobench', dataset_args=dataset_args, limit=5, eval_batch_size=1)

    def test_science_qa_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('science_qa', dataset_args=dataset_args, limit=10)

    def test_cmmu_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('cmmu', dataset_args=dataset_args, use_cache='outputs/20251112_163342', limit=10, rerun_review=True, eval_batch_size=1)

    def test_a_okvqa_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('a_okvqa', dataset_args=dataset_args, limit=10)

    def test_vstar_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('vstar_bench', dataset_args=dataset_args, limit=10)

    def test_micro_vqa_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('micro_vqa', dataset_args=dataset_args, limit=10)

    def test_gsm8k_v(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('gsm8k_v', dataset_args=dataset_args, limit=20)

    def test_fleurs(self):
        dataset_args = {
            'subset_list': [
                'cmn_hans_cn',
                'en_us',
                # 'yue_hant_hk',
            ]
        }
        self._run_dataset_test('fleurs', dataset_args=dataset_args, limit=100, model='qwen3-omni-flash', use_cache='outputs/20251209_110609', ignore_errors=True, rerun_review=True)

    def test_librispeech(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('librispeech', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash', ignore_errors=True, rerun_review=True)

    def test_refcoco_load(self):
        dataset_args = {
            'subset_list': ['testA'],
            'extra_params': {
                'eval_mode': 'bbox_rec'
            }
        }
        self._run_dataset_load_test('refcoco', dataset_args=dataset_args)

    def test_refcoco_bbox(self):
        dataset_args = {
            'subset_list': ['testA'],
            'extra_params': {
                'eval_mode': 'bbox'
            }
        }
        self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=5)

    def test_refcoco_seg(self):
        dataset_args = {
            'shuffle': True,
            'subset_list': ['testA'],
            'extra_params': {
                'eval_mode': 'seg',
            }
        }
        self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=10)

    def test_refcoco_bbox_rec(self):
        dataset_args = {
            'subset_list': ['testA'],
            'extra_params': {
                'eval_mode': 'bbox_rec'
            }
        }
        self._run_dataset_test('refcoco', dataset_args=dataset_args, limit=10)

    def test_mia_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        # self._run_dataset_load_test('mia_bench')
        self._run_dataset_test('mia_bench', dataset_args=dataset_args, limit=10)

    def test_tir_bench(self):
        dataset_args = {
            # 'subset_list': ['default']
        }
        self._run_dataset_test('tir_bench', dataset_args=dataset_args, limit=5, ignore_errors=True, use_cache='outputs/20260415_173317', rerun_review=True)

    def test_tir_bench_load(self):
        self._run_dataset_load_test('tir_bench')

    def test_air_bench_foundation(self):
        dataset_args = {
            # 'subset_list': ['Music_AQA_music_avqa'],
        }
        self._run_dataset_test('air_bench_foundation', dataset_args=dataset_args, limit=2, model='qwen3-omni-flash', ignore_errors=True)

    def test_air_bench_chat(self):
        dataset_args = {
            'extra_params': {
                'tasks': ['speech_QA'],
                'do_swap': False,
            },
        }
        self._run_dataset_test('air_bench_chat', dataset_args=dataset_args, limit=2, model='qwen3-omni-flash', ignore_errors=True)

    def test_mvbench(self):
        dataset_args={
            'subset_list': ['action_antonym'],
        }
        self._run_dataset_test('mvbench', dataset_args=dataset_args, limit=10)

    def test_videomme_v2(self):
        dataset_args = {
            'subset_list': ['level_1']
        }
        self._run_dataset_test('videomme_v2', dataset_args=dataset_args, limit=10)

    def test_common_voice_15(self):
        dataset_args = {
            'subset_list': ['en'],
        }
        self._run_dataset_test('common_voice_15', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')

    def test_wenet_speech(self):
        dataset_args = {
            'subset_list': ['dev'],
        }
        self._run_dataset_test('wenet_speech', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')

    def test_mmau(self):
        dataset_args = {
            'subset_list': ['test_mini'],
        }
        self._run_dataset_test('mmau', dataset_args=dataset_args, limit=5, model='qwen3-omni-flash')

    def test_erqa(self):
        self._run_dataset_test('erqa', limit=5)

    def test_world_vqa(self):
        self._run_dataset_test('world_vqa', limit=5)

    def test_charxiv(self):
        """Test CharXiv chart understanding benchmark."""
        self._run_dataset_test('charxiv', limit=5)

    def test_baby_vision(self):
        """Test BabyVision visual perception benchmark."""
        self._run_dataset_test('baby_vision', limit=5)