sora/evalstone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

xii

Browse Source
This commit is contained in:
aaxiyao 2026-07-10 04:02:28 +00:00
parent d7547f642a
commit 85e1a9796d
3 changed files with 686 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
bash/generation_config_nothinking.yaml Normal file
View File

@ -0,0 +1,249 @@
terminal_bench_v2:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
max_steps: 500
per_step_tokens: 49152
gpqa_diamond:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
hle:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
aime24:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
aime25:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
mmlu_pro:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
simple_qa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
arc:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
bbh:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
browsecomp:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 49152
max_steps: 500
live_code_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
dataset_args:
subset_list:
- release_v6
swe_bench_pro:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
aime26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
hmmt26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
imo_answerbench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
max_steps: 500
super_gpqa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
drop:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
hellaswag:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
mmlu:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
openai_mrcr:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
mcp_atlas:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 49152
max_steps: 500
swe_bench_multilingual_agentic:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
swe_bench_verified:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
bigcodebench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
humaneval:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
gsm8k:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
competition_math:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
cmmlu:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
trivia_qa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
winogrande:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
longbench_v2:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
tau2_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500

249
bash/generation_config_thinking.yaml Normal file
View File

@ -0,0 +1,249 @@
terminal_bench_v2:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
max_steps: 500
per_step_tokens: 49152
gpqa_diamond:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
hle:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
aime24:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
aime25:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
mmlu_pro:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
simple_qa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
arc:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
bbh:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
browsecomp:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 49152
max_steps: 500
live_code_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
dataset_args:
subset_list:
- release_v6
swe_bench_pro:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
aime26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
hmmt26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
imo_answerbench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
max_steps: 500
super_gpqa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
drop:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
hellaswag:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
mmlu:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
openai_mrcr:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
mcp_atlas:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 49152
max_steps: 500
swe_bench_multilingual_agentic:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
swe_bench_verified:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500
bigcodebench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
humaneval:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
gsm8k:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
competition_math:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
cmmlu:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
trivia_qa:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
winogrande:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
longbench_v2:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 131072
tau2_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 524288
agent_config:
per_step_tokens: 32768
max_steps: 500

188
bash/run.py Normal file
View File

@ -0,0 +1,188 @@
# cd /data1/p800_deploy_scripts/DeepSeek-V4-Flash-xsglang-20260511-deploy
# docker rm -f /deepseek-v4-flash-xsglang-0511
# bash /data1/p800_deploy_scripts/DeepSeek-V4-Flash-xsglang-20260511-deploy/scripts/docker_run.sh
# bash scripts/run_server.sh
import os
import yaml
from copy import deepcopy
from evalscope import run_task, TaskConfig
from evalscope.api.agent import NativeAgentConfig
# ============================================================
# 1. 用户可配置项
# ============================================================
CONFIG_PATH = '/data1/benchmark/evalscope/bash/generation_config_nothinking.yaml'
# 显式开关：True 表示开启模型的 thinking 能力，False 表示关闭
ENABLE_THINKING = False
# 选择本次实际要跑的数据集
DATASETS = [
# 'aime26',
# 'terminal_bench_v2',
# 'gpqa_diamond',
# 'hle',
# 'aime24',
# 'aime25',
# 'mmlu_pro',
# 'simple_qa',
# 'arc',
# 'bbh',
# 'browsecomp',
# 'live_code_bench',
# 'swe_bench_pro',
# 'hmmt26',
# 'imo_answerbench',
# 'super_gpqa',
# 'drop',
# 'hellaswag',
# 'mmlu',
# 'openai_mrcr',
# 'mcp_atlas',
# 'swe_bench_multilingual_agentic',
# 'swe_bench_verified',
'bigcodebench',
# 'humaneval',
# 'gsm8k',
# 'competition_math',
# 'cmmlu',
# 'trivia_qa',
# 'winogrande',
# 'longbench_v2',
# 'tau2_bench',
]
# 公共配置（所有数据集共享）
COMMON_CONFIG = {
'seed': 42,
'collect_perf': True,
'no_timestamp': True,
'model': 'DeepSeek-V4-Flash-Int8',
'api_url': 'http://localhost:30000/v1',
# 'api_key': '',
'eval_type': 'openai_api',
'dataset_dir': '/data1/benchmark/bash/datasets',
'judge_model_args': {
'model_id': 'deepseek-v4-pro',
'api_url': 'https://api.deepseek.com/v1',
'api_key': 'sk-9ed86ef546ca47e3afa7c3b014dea268',
'eval_type': 'openai_api',
'generation_config': {
'temperature': 0.0,
'max_tokens': 1024 * 10,
},
},
}
# 数学类任务集合
MATH_DATASETS = {
'aime24', 'aime25', 'aime26', 'hmmt26',
'gsm8k', 'competition_math', 'imo_answerbench',
}
# 数学类任务 prompt 模板
MATH_PROMPT_TEMPLATE = (
"{question}\n"
"Please reason step by step, and put your final answer within \\boxed{{}}."
)
batch_size_list = [16]
# ============================================================
# 2. 加载 YAML 配置
# ============================================================
with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
DATASET_CONFIGS = yaml.safe_load(f)
# ============================================================
# 3. 辅助函数
# ============================================================
def configure_thinking(generation_config: dict, enable: bool) -> dict:
"""显式开启或关闭 generation_config 中的 thinking 能力。"""
if enable:
generation_config['thinking'] = {'type': 'enabled'}
else:
generation_config.pop('thinking', None)
return generation_config
def build_agent_config(agent_cfg: dict) -> NativeAgentConfig:
"""从 YAML 的 agent_config 构造 NativeAgentConfig。
YAML 里可能包含 strategy 未声明的字段（如 per_step_tokens），
这些字段会被放到 kwargs 里，透传给 Agent Strategy。
"""
agent_cfg = deepcopy(agent_cfg or {})
# NativeAgentConfig 已知的顶层字段
known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'}
kwargs = agent_cfg.pop('kwargs', {})
# 把未知字段收敛进 kwargs
for key in list(agent_cfg.keys()):
if key not in known_fields:
kwargs[key] = agent_cfg.pop(key)
if kwargs:
agent_cfg['kwargs'] = kwargs
return NativeAgentConfig(**agent_cfg)
def build_task_config(dataset_name: str, ds_cfg: dict, batch_size: int, enable_thinking: bool) -> TaskConfig:
"""根据数据集配置构造单个 TaskConfig。"""
work_dir = f'/data1/benchmark/temp/output_{batch_size}/{dataset_name}'
# 构造 generation_config，并显式处理 thinking 开关
generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
# 构造 dataset_args：仅对应当前数据集
dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
# 数学类任务覆盖 prompt_template
if dataset_name in MATH_DATASETS:
dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE
dataset_args_dict = {dataset_name: dataset_args}
# 构造 agent_config（如果 YAML 里配了的话）
agent_config = None
if 'agent_config' in ds_cfg:
agent_config = build_agent_config(ds_cfg['agent_config'])
return TaskConfig(
**COMMON_CONFIG,
work_dir=work_dir,
use_cache=work_dir,
datasets=[dataset_name],
generation_config=generation_config,
dataset_args=dataset_args_dict,
agent_config=agent_config,
eval_batch_size=batch_size,
)
# ============================================================
# 4. 主循环
# ============================================================
if __name__ == '__main__':
for batch_size in batch_size_list:
task_cfg_list = []
for dataset_name in DATASETS:
if dataset_name not in DATASET_CONFIGS:
raise ValueError(
f"Dataset '{dataset_name}' 不在 {CONFIG_PATH} 中，"
f"请先在 YAML 里补充它的 generation_config。"
)
ds_cfg = DATASET_CONFIGS[dataset_name]
task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING)
task_cfg_list.append(task_cfg)
# run_task 支持 list，会串行执行每个数据集的评测
run_task(task_cfg_list)