diff --git a/bash/run.py b/bash/run.py index f41870f..61e9796 100644 --- a/bash/run.py +++ b/bash/run.py @@ -151,14 +151,13 @@ SUITES = { # 腾讯/官方入库标准对比套件:取各厂商公开对比表的并集,约 11 个 benchmark # 注:swe_bench_verified/terminal_bench_v2/browsecomp/tau2_bench 需要额外镜像/工具 'official': { - 'multi': [], + 'multi': ['aime25', 'aime26','live_code_bench'], 'single': [ - 'aime25', 'aime26', - 'hle', 'mmlu_pro', 'gpqa_diamond', - 'longbench_v2', 'live_code_bench', 'swe_bench_verified', + 'hle', 'mmlu_pro', 'gpqa_diamond','longbench_v2', + # 'swe_bench_verified', ], 'agent': [ - 'tau2_bench', 'terminal_bench_v2', 'browsecomp', + 'tau2_bench' ], }, } diff --git a/myread.md b/myread.md index be5a666..260996b 100644 --- a/myread.md +++ b/myread.md @@ -409,7 +409,7 @@ python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none | `group1` | 代码 + 数学/推理 | 约 **22h** | 多机组 1 | | `group2` | 重知识 | 约 **26h** | 多机组 2 | | `group3` | 长文本 + Agent + 知识 | 约 **27h** | 多机组 3 | -| `official` | `aime25`, `aime26`, `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `live_code_bench`, `swe_bench_verified`, `tau2_bench`, `terminal_bench_v2`, `browsecomp` | 视配置而定 | 腾讯入库标准对比表的并集,约 11 个 benchmark;其中 swe/terminal/browsecomp/tau2 需额外镜像/工具 | +| `official` | `aime25`, `aime26`, `live_code_bench`, `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `tau2_bench` | 视配置而定 | 腾讯入库标准对比表核心子集;aime/live_code_bench 多次采样,tau2_bench 为 Agent 任务 | > **分组时间说明** >