13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
67 lines
2.1 KiB
Python
67 lines
2.1 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
from dotenv import dotenv_values
|
|
|
|
env = dotenv_values('.env')
|
|
|
|
import unittest
|
|
|
|
from evalscope.constants import EvalType, JudgeStrategy, ModelTask
|
|
from evalscope.utils.logger import get_logger
|
|
from tests.common import TestBenchmark
|
|
|
|
logger = get_logger()
|
|
|
|
|
|
class TestImageEditBenchmark(TestBenchmark):
|
|
def setUp(self):
|
|
"""Setup common test configuration."""
|
|
self.base_config = {
|
|
'model': 'Qwen/Qwen-Image-Edit',
|
|
'model_args':{
|
|
'precision': 'bfloat16',
|
|
'device_map': 'cuda:2'
|
|
},
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'model_task': ModelTask.IMAGE_GENERATION,
|
|
'eval_type': EvalType.IMAGE_EDITING,
|
|
'eval_batch_size': 5,
|
|
'limit': 5,
|
|
'generation_config': {
|
|
'true_cfg_scale': 4.0,
|
|
'num_inference_steps': 50,
|
|
'negative_prompt': ' ',
|
|
},
|
|
'judge': {
|
|
'strategy': JudgeStrategy.AUTO,
|
|
'models': {
|
|
'model_id': 'qwen-plus',
|
|
'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
'api_key': env.get('DASHSCOPE_API_KEY'),
|
|
'generation_config': {
|
|
'temperature': 0.0,
|
|
'max_tokens': 4096,
|
|
}
|
|
},
|
|
},
|
|
'debug': True,
|
|
}
|
|
|
|
def test_gedit(self):
|
|
"""Test GEdit dataset."""
|
|
dataset_args = {
|
|
'extra_params':{
|
|
'language': 'cn',
|
|
}
|
|
}
|
|
self._run_dataset_test('gedit', dataset_args=dataset_args, use_cache='outputs/20250829_150058')
|
|
|
|
def test_gedit_local(self):
|
|
dataset_args = {
|
|
'extra_params':{
|
|
'language': 'cn',
|
|
'local_file': 'outputs/example_edit.jsonl',
|
|
}
|
|
}
|
|
self._run_dataset_test('gedit', dataset_args=dataset_args, model=None, model_id='offline_model')
|