gpqa_diamond:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
hle:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
aime24:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
aime25:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
mmlu_pro:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
simple_qa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
arc:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
bbh:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
live_code_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    subset_list:
    - release_v6
aime26:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
hmmt26:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
imo_answerbench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
super_gpqa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
drop:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
hellaswag:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
mmlu:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
openai_mrcr:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
bigcodebench:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
humaneval:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
gsm8k:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
competition_math:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
cmmlu:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
trivia_qa:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
winogrande:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
longbench_v2:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
  dataset_args:
    subset_list:
    - short
    - medium
    - long
tau2_bench:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 16384
  dataset_args:
    extra_params:
      user_model: deepseek-v4-pro
      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
      api_base: https://api.deepseek.com/v1
      generation_config:
        temperature: 0.0
        max_tokens: 4096
  agent_config:
    mode: native
    strategy: react
    max_steps: 50
general_fc:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 4096
bfcl_v3:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 4096
    parallel_tool_calls: true
  dataset_args:
    extra_params:
      is_fc_model: true
      underscore_to_dot: true
swe_bench_verified:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
swe_bench_pro:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
terminal_bench_v2_1:
  generation_config:
    temperature: 0.0
    top_p: 1.0
    stream: true
    max_tokens: 8192
  dataset_args:
    extra_params:
      timeout_multiplier: 2.0
      max_turns: 500
aa_lcr:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
scicode:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
deep_swe:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    extra_params:
      pier_model_prefix: openai
      pier_agent_kwargs:
        model_class: litellm
researchrubrics:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
gdpval:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
tau3_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 16384
  dataset_args:
    subset_list:
    - banking_knowledge
    extra_params:
      user_model: deepseek-v4-pro
      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
      api_base: https://api.deepseek.com/v1
      generation_config:
        temperature: 0.0
        max_tokens: 4096
toolathlon:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    extra_params:
      server_host: 127.0.0.1
      server_port: 8080
      ws_proxy_port: 8081
      timeout_seconds: 14400
      submit_timeout_seconds: 300
      task_list:
      - paper-checker
      skip_container_restart: false
      override_output_dir: true
browsecomp:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
mcp_atlas:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
officeqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    subset_list:
    - officeqa_pro
deepsearchqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
job_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
automation_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    extra_params:
      toolset: api
mmmu_pro:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    max_image_bytes: 5mb
    extra_params:
      dataset_format: standard (4 options)
charxiv:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    subset_list:
    - reasoning
math_vision:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
baby_vision:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
zerobench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    max_image_bytes: 10mb
world_vqa:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
omni_doc_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768
  dataset_args:
    extra_params:
      match_method: quick_match
perception_bench:
  generation_config:
    temperature: 1.0
    top_p: 1.0
    stream: true
    max_tokens: 32768