diff --git a/bash/run.py b/bash/run.py index 0b229ca..f41870f 100644 --- a/bash/run.py +++ b/bash/run.py @@ -148,22 +148,17 @@ SUITES = { 'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'], 'agent': ['tau2_bench', 'general_fc'], }, - # 腾讯/官方入库标准对比套件:覆盖各厂商公开对比表里的全部 benchmark - # 注:swe_bench_*/terminal_bench_v2/browsecomp/mcp_atlas 需要额外镜像/工具 + # 腾讯/官方入库标准对比套件:取各厂商公开对比表的并集,约 11 个 benchmark + # 注:swe_bench_verified/terminal_bench_v2/browsecomp/tau2_bench 需要额外镜像/工具 'official': { - 'multi': [ - 'aime24', 'aime25', 'aime26', 'hmmt26', - 'imo_answerbench', 'live_code_bench', 'humaneval', 'gpqa_diamond', - ], + 'multi': [], 'single': [ - 'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa', - 'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro', - 'simple_qa', 'trivia_qa', 'winogrande', 'openai_mrcr', 'longbench_v2', - 'swe_bench_verified', 'swe_bench_pro', 'swe_bench_multilingual_agentic', - 'terminal_bench_v2', + 'aime25', 'aime26', + 'hle', 'mmlu_pro', 'gpqa_diamond', + 'longbench_v2', 'live_code_bench', 'swe_bench_verified', ], 'agent': [ - 'tau2_bench', 'tau2_bench_retail', 'browsecomp', 'mcp_atlas', 'general_fc', + 'tau2_bench', 'terminal_bench_v2', 'browsecomp', ], }, } diff --git a/myread.md b/myread.md index 951561d..be5a666 100644 --- a/myread.md +++ b/myread.md @@ -409,7 +409,7 @@ python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none | `group1` | 代码 + 数学/推理 | 约 **22h** | 多机组 1 | | `group2` | 重知识 | 约 **26h** | 多机组 2 | | `group3` | 长文本 + Agent + 知识 | 约 **27h** | 多机组 3 | -| `official` | 官方对比表全部 benchmark | 视配置而定 | 覆盖 Kimi/GLM/DS 等公开对比表里的所有任务 | +| `official` | `aime25`, `aime26`, `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `live_code_bench`, `swe_bench_verified`, `tau2_bench`, `terminal_bench_v2`, `browsecomp` | 视配置而定 | 腾讯入库标准对比表的并集,约 11 个 benchmark;其中 swe/terminal/browsecomp/tau2 需额外镜像/工具 | > **分组时间说明** >