Update official suite: aime25/aime26/live_code_bench multi-run + core single/agent benchmarks
This commit is contained in:
parent
9755c2a5a0
commit
69b240ad6d
@ -151,14 +151,13 @@ SUITES = {
|
||||
# 腾讯/官方入库标准对比套件:取各厂商公开对比表的并集,约 11 个 benchmark
|
||||
# 注:swe_bench_verified/terminal_bench_v2/browsecomp/tau2_bench 需要额外镜像/工具
|
||||
'official': {
|
||||
'multi': [],
|
||||
'multi': ['aime25', 'aime26','live_code_bench'],
|
||||
'single': [
|
||||
'aime25', 'aime26',
|
||||
'hle', 'mmlu_pro', 'gpqa_diamond',
|
||||
'longbench_v2', 'live_code_bench', 'swe_bench_verified',
|
||||
'hle', 'mmlu_pro', 'gpqa_diamond','longbench_v2',
|
||||
# 'swe_bench_verified',
|
||||
],
|
||||
'agent': [
|
||||
'tau2_bench', 'terminal_bench_v2', 'browsecomp',
|
||||
'tau2_bench'
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@ -409,7 +409,7 @@ python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none
|
||||
| `group1` | 代码 + 数学/推理 | 约 **22h** | 多机组 1 |
|
||||
| `group2` | 重知识 | 约 **26h** | 多机组 2 |
|
||||
| `group3` | 长文本 + Agent + 知识 | 约 **27h** | 多机组 3 |
|
||||
| `official` | `aime25`, `aime26`, `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `live_code_bench`, `swe_bench_verified`, `tau2_bench`, `terminal_bench_v2`, `browsecomp` | 视配置而定 | 腾讯入库标准对比表的并集,约 11 个 benchmark;其中 swe/terminal/browsecomp/tau2 需额外镜像/工具 |
|
||||
| `official` | `aime25`, `aime26`, `live_code_bench`, `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `tau2_bench` | 视配置而定 | 腾讯入库标准对比表核心子集;aime/live_code_bench 多次采样,tau2_bench 为 Agent 任务 |
|
||||
|
||||
> **分组时间说明**
|
||||
>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user