gpqa_diamond:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
hle:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
aime24:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
aime25:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 65536
    max_completion_tokens: 64000
mmlu_pro:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
simple_qa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
arc:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
bbh:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
live_code_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 65536
    max_completion_tokens: 64000
  dataset_args:
    subset_list:
    - release_v6
aime26:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 65536
    max_completion_tokens: 64000
hmmt26:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
imo_answerbench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
super_gpqa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
drop:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
hellaswag:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
mmlu:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
openai_mrcr:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
bigcodebench:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
humaneval:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
gsm8k:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
competition_math:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
cmmlu:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
trivia_qa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
winogrande:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
longbench_v2:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
    max_completion_tokens: 64000
  dataset_args:
    subset_list:
    - short
    - medium
    - long
tau2_bench:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 16384
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      user_model: deepseek-v4-pro
      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
      api_base: https://api.deepseek.com/v1
      generation_config:
        temperature: 0.0
        max_tokens: 4096
        max_completion_tokens: 64000
  agent_config:
    mode: native
    strategy: react
    max_steps: 50
general_fc:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 4096
    max_completion_tokens: 64000
bfcl_v3:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 4096
    max_completion_tokens: 64000
    parallel_tool_calls: true
  dataset_args:
    extra_params:
      is_fc_model: true
      underscore_to_dot: true
swe_bench_verified:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
swe_bench_pro:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
aa_lcr:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
scicode:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
deep_swe:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      pier_model_prefix: openai
      pier_agent_kwargs:
        model_class: litellm
researchrubrics:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
gdpval:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
tau3_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 16384
    max_completion_tokens: 64000
  dataset_args:
    subset_list:
    - banking_knowledge
    extra_params:
      user_model: deepseek-v4-pro
      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
      api_base: https://api.deepseek.com/v1
      generation_config:
        temperature: 0.0
        max_tokens: 4096
        max_completion_tokens: 64000
toolathlon:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      server_host: 127.0.0.1
      server_port: 8080
      ws_proxy_port: 8081
      timeout_seconds: 14400
      submit_timeout_seconds: 300
      task_list:
      - paper-checker
      skip_container_restart: false
      override_output_dir: true
terminal_bench_v2_1:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      timeout_multiplier: 2.0
      max_turns: 500
browsecomp:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
mcp_atlas:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
officeqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    subset_list:
    - officeqa_pro
deepsearchqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
job_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
automation_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      toolset: api
mmmu_pro:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    max_image_bytes: 5mb
    extra_params:
      dataset_format: standard (4 options)
charxiv:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    subset_list:
    - reasoning
math_vision:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
baby_vision:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
zerobench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    max_image_bytes: 10mb
world_vqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
omni_doc_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000
  dataset_args:
    extra_params:
      match_method: quick_match
perception_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
    max_completion_tokens: 64000