29 lines
1.8 KiB
Python
29 lines
1.8 KiB
Python
from evalscope.collections import CollectionSchema, DatasetInfo, WeightedSampler
|
|
from evalscope.utils.io_utils import dump_jsonl_data
|
|
|
|
schema = CollectionSchema(name='Qwen3', datasets=[
|
|
CollectionSchema(name='English', datasets=[
|
|
DatasetInfo(name='mmlu_pro', weight=1, task_type='exam', tags=['en'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='mmlu_redux', weight=1, task_type='exam', tags=['en'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='ifeval', weight=1, task_type='instruction', tags=['en'], args={'few_shot_num': 0}),
|
|
]),
|
|
CollectionSchema(name='Chinese', datasets=[
|
|
DatasetInfo(name='ceval', weight=1, task_type='exam', tags=['zh'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='iquiz', weight=1, task_type='exam', tags=['zh'], args={'few_shot_num': 0}),
|
|
]),
|
|
CollectionSchema(name='Code', datasets=[
|
|
DatasetInfo(name='live_code_bench', weight=1, task_type='code', tags=['en'], args={'few_shot_num': 0, 'subset_list': ['v5_v6'], 'extra_params': {'start_date': '2025-01-01', 'end_date': '2025-04-30'}}),
|
|
]),
|
|
CollectionSchema(name='Math&Science', datasets=[
|
|
DatasetInfo(name='math_500', weight=1, task_type='math', tags=['en'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='aime24', weight=1, task_type='math', tags=['en'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='aime25', weight=1, task_type='math', tags=['en'], args={'few_shot_num': 0}),
|
|
DatasetInfo(name='gpqa_diamond', weight=1, task_type='knowledge', tags=['en'], args={'few_shot_num': 0})
|
|
])
|
|
])
|
|
|
|
# get the mixed data
|
|
mixed_data = WeightedSampler(schema).sample(100000000) # set a large number to ensure all datasets are sampled
|
|
# dump the mixed data to a jsonl file
|
|
dump_jsonl_data(mixed_data, 'outputs/qwen3_test.jsonl')
|