105 lines
3.3 KiB
Python
105 lines
3.3 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
from dotenv import dotenv_values, load_dotenv
|
|
|
|
load_dotenv('.env')
|
|
|
|
env = dotenv_values('.env')
|
|
|
|
import unittest
|
|
|
|
from evalscope.constants import EvalType, JudgeStrategy, OutputType
|
|
from evalscope.utils.logger import get_logger
|
|
from tests.common import TestBenchmark
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
class TestModels(TestBenchmark):
|
|
"""Benchmark evaluation test cases."""
|
|
|
|
def setUp(self):
|
|
"""Setup common test configuration."""
|
|
self.base_config = {
|
|
'model': 'Qwen2.5-VL-7B-Instruct',
|
|
'api_url': 'http://localhost:8000/',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'eval_type': EvalType.ANTHROPIC_API, # test anthropic api
|
|
'eval_batch_size': 5,
|
|
'limit': 5,
|
|
'generation_config': {
|
|
'max_tokens': 4096,
|
|
'temperature': 0.7,
|
|
'parallel_tool_calls': True,
|
|
'stream': True
|
|
},
|
|
'judge_strategy': JudgeStrategy.AUTO,
|
|
'judge_model_args': {
|
|
'model_id': 'qwen3-235b-a22b',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'generation_config': {
|
|
'temperature': 0.0,
|
|
'max_tokens': 4096,
|
|
'extra_body': {'enable_thinking': False}
|
|
}
|
|
},
|
|
'debug': True,
|
|
}
|
|
|
|
|
|
# Math & Reasoning datasets
|
|
def test_gsm8k(self):
|
|
"""Test GSM8K math reasoning dataset."""
|
|
dataset_args = {
|
|
'few_shot_num': 0,
|
|
}
|
|
self._run_dataset_test('gsm8k', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_chartqa(self):
|
|
"""Test ChartQA math reasoning dataset."""
|
|
dataset_args = {
|
|
'few_shot_num': 0,
|
|
}
|
|
self._run_dataset_test('chartqa', dataset_args=dataset_args, limit=5)
|
|
|
|
def test_bfcl_v4(self):
|
|
"""Test BFCL-V4 math reasoning dataset."""
|
|
"""Test BFCL v4 dataset."""
|
|
dataset_args = {
|
|
'subset_list': [
|
|
'simple_python',
|
|
# 'simple_java',
|
|
# 'simple_javascript',
|
|
# 'multiple',
|
|
# 'parallel',
|
|
# 'parallel_multiple',
|
|
# 'irrelevance',
|
|
# 'live_simple',
|
|
# 'live_multiple',
|
|
# 'live_parallel',
|
|
# 'live_parallel_multiple',
|
|
# 'live_irrelevance',
|
|
# 'live_relevance',
|
|
# 'multi_turn_base',
|
|
# 'multi_turn_miss_func',
|
|
# 'multi_turn_miss_param',
|
|
# 'multi_turn_long_context',
|
|
# 'web_search_base',
|
|
# 'web_search_no_snippet',
|
|
# 'memory_kv',
|
|
# 'memory_vector',
|
|
# 'memory_rec_sum'
|
|
],
|
|
'extra_params': {
|
|
'is_fc_model': True,
|
|
'underscore_to_dot': True,
|
|
'SERPAPI_API_KEY':env.get('SERPAPI_API_KEY'),
|
|
}
|
|
}
|
|
self._run_dataset_test('bfcl_v4', dataset_args=dataset_args)
|
|
|
|
def test_chartqa(self):
|
|
"""Test ChartQA math reasoning dataset."""
|
|
|
|
self._run_dataset_test('chartqa')
|