f662006517
Ports es's swe_bench_agentic_adapter into our plugin architecture: - env swe_agentic: per-sample LONG-RUNNING container (official sweb image, /testbed, bash -lc like the testbed startup files expect), single bash tool via function calling, sentinel-submission protocol (COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT + patch), git-diff fallback; observations capped at 30k chars - dataset swe_bench_verified_agentic: same princeton source/converter, separate bench name so both variants coexist - recipe: recovered patch + OFFICIAL test_patch applied in-container, FAIL_TO_PASS + capped PASS_TO_PASS via conda testbed pytest, 1800s - config: max_turns 250, env swe_agentic Single-turn swe_bench_verified is untouched. Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
1.6 KiB
YAML
75 lines
1.6 KiB
YAML
default:
|
||
temperature: 0.0
|
||
top_p: 1.0
|
||
stream: true
|
||
max_tokens: 32768
|
||
max_turns: 200
|
||
aime24:
|
||
temperature: 1.0
|
||
repeats: 12
|
||
max_tokens: 8192
|
||
aime25:
|
||
temperature: 1.0
|
||
repeats: 12
|
||
max_tokens: 8192
|
||
aime26:
|
||
temperature: 1.0
|
||
repeats: 12
|
||
max_tokens: 8192
|
||
hmmt26:
|
||
temperature: 1.0
|
||
repeats: 12
|
||
max_tokens: 8192
|
||
imo_answerbench:
|
||
temperature: 1.0
|
||
gpqa_diamond:
|
||
temperature: 1.0
|
||
repeats: 3
|
||
max_tokens: 8192
|
||
mmlu:
|
||
max_tokens: 8192
|
||
mmlu_pro:
|
||
max_tokens: 8192
|
||
cmmlu:
|
||
max_tokens: 8192
|
||
arc:
|
||
max_tokens: 8192
|
||
hellaswag:
|
||
max_tokens: 8192
|
||
winogrande:
|
||
max_tokens: 8192
|
||
simple_qa:
|
||
max_tokens: 8192
|
||
trivia_qa:
|
||
max_tokens: 8192
|
||
humaneval:
|
||
temperature: 1.0
|
||
repeats: 3
|
||
live_code_bench:
|
||
temperature: 1.0
|
||
longbench_v2:
|
||
max_tokens: 8192
|
||
max_input_tokens: 128000
|
||
openai_mrcr:
|
||
max_tokens: 8192
|
||
max_input_tokens: 128000
|
||
bfcl_v3:
|
||
max_tokens: 4096
|
||
env: bfcl_mock # agent 模式：工具调用轨迹 + 官方 AST 判分
|
||
general_fc:
|
||
max_tokens: 4096
|
||
tau2_bench:
|
||
max_tokens: 16384
|
||
env: tau2_official # agent 模式：官方引擎
|
||
max_turns: 200 # 一次完整电话流程（问候→验证→找单→政策→执行→确认）
|
||
|
||
swe_bench_verified:
|
||
max_tokens: 32768 # patch 生成预算（diff 可能较长）
|
||
temperature: 0.0 # es 口径：确定性生成 patch
|
||
|
||
swe_bench_verified_agentic:
|
||
max_tokens: 4096 # 单轮 bash 命令生成预算（mini-swe-agent 口径）
|
||
temperature: 0.0
|
||
env: swe_agentic # 多轮 agent：bash 探索 /testbed + sentinel 提交
|
||
max_turns: 250 # mini-swe-agent 默认步数
|