250 lines
3.9 KiB
YAML
250 lines
3.9 KiB
YAML
terminal_bench_v2:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
max_steps: 500
|
|
per_step_tokens: 49152
|
|
|
|
gpqa_diamond:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
hle:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
aime24:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
aime25:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
mmlu_pro:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
simple_qa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
arc:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
bbh:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
browsecomp:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 49152
|
|
max_steps: 500
|
|
|
|
live_code_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
dataset_args:
|
|
subset_list:
|
|
- release_v6
|
|
|
|
swe_bench_pro:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 32768
|
|
max_steps: 500
|
|
|
|
aime26:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
hmmt26:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
imo_answerbench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
max_steps: 500
|
|
|
|
super_gpqa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
drop:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
hellaswag:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
mmlu:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
openai_mrcr:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
mcp_atlas:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 49152
|
|
max_steps: 500
|
|
|
|
swe_bench_multilingual_agentic:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 32768
|
|
max_steps: 500
|
|
|
|
swe_bench_verified:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 32768
|
|
max_steps: 500
|
|
|
|
bigcodebench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
|
|
humaneval:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
|
|
gsm8k:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
competition_math:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 131072
|
|
|
|
cmmlu:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
trivia_qa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
winogrande:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
longbench_v2:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
|
|
tau2_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 524288
|
|
agent_config:
|
|
per_step_tokens: 32768
|
|
max_steps: 500
|