default:
  temperature: 0.0
  top_p: 1.0
  stream: true
  max_tokens: 32768
  concurrency: 8

aime24:
  temperature: 1.0
  repeats: 12

aime25:
  temperature: 1.0
  repeats: 12

aime26:
  temperature: 1.0
  repeats: 12

hmmt26:
  temperature: 1.0
  repeats: 12

imo_answerbench:
  temperature: 1.0
  limit_per_task: 25
  judge: dp4-flash
  judge_url: http://174.1.51.4:30000/v1

gsm8k:
  limit: 200

competition_math:
  limit_per_task: 40

mmlu:
  max_tokens: 8192
  limit_per_task: 4

cmmlu:
  max_tokens: 8192
  limit_per_task: 3

mmlu_pro:
  max_tokens: 8192
  limit_per_task: 20

arc:
  max_tokens: 8192
  limit: 200

hellaswag:
  max_tokens: 8192
  limit: 400

winogrande:
  max_tokens: 8192
  limit: 200

bbh:
  limit_per_task: 10

gpqa_diamond:
  temperature: 1.0
  max_tokens: 8192
  limit: 200

simple_qa:
  max_tokens: 8192
  limit: 200

trivia_qa:
  max_tokens: 8192
  limit: 200

drop:
  limit: 200

hle:
  limit_per_task: 25
  judge: dp4-flash
  judge_url: http://174.1.51.4:30000/v1

humaneval:
  temperature: 1.0

bigcodebench:
  limit: 200

live_code_bench:
  temperature: 1.0
  limit: 200

longbench_v2:
  max_tokens: 8192
  limit_per_task: 66
  max_input_tokens: 128000

openai_mrcr:
  max_tokens: 8192
  limit: 200
  max_input_tokens: 128000

bfcl_v3:
  max_tokens: 4096
  limit_per_task: 20
  env: bfcl_mock

general_fc:
  max_tokens: 4096
  limit: 400

tau2_bench:
  max_tokens: 16384
  env: tau2_official
  max_turns: 50

swe_bench_verified:
  limit: 70
  concurrency: 4