13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
440 lines
8.8 KiB
YAML
440 lines
8.8 KiB
YAML
gpqa_diamond:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
hle:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
aime24:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
aime25:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 65536
|
|
max_completion_tokens: 64000
|
|
mmlu_pro:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
simple_qa:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
arc:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
bbh:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
live_code_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 65536
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
subset_list:
|
|
- release_v6
|
|
aime26:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 65536
|
|
max_completion_tokens: 64000
|
|
hmmt26:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
imo_answerbench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
super_gpqa:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
drop:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
hellaswag:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
mmlu:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
openai_mrcr:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
bigcodebench:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
humaneval:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
gsm8k:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
competition_math:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
cmmlu:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
trivia_qa:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
winogrande:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
longbench_v2:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 8192
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
subset_list:
|
|
- short
|
|
- medium
|
|
- long
|
|
tau2_bench:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 16384
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
user_model: deepseek-v4-pro
|
|
api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
|
|
api_base: https://api.deepseek.com/v1
|
|
generation_config:
|
|
temperature: 0.0
|
|
max_tokens: 4096
|
|
max_completion_tokens: 64000
|
|
agent_config:
|
|
mode: native
|
|
strategy: react
|
|
max_steps: 50
|
|
general_fc:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 4096
|
|
max_completion_tokens: 64000
|
|
bfcl_v3:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 4096
|
|
max_completion_tokens: 64000
|
|
parallel_tool_calls: true
|
|
dataset_args:
|
|
extra_params:
|
|
is_fc_model: true
|
|
underscore_to_dot: true
|
|
swe_bench_verified:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
swe_bench_pro:
|
|
generation_config:
|
|
temperature: 0.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
aa_lcr:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
scicode:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
deep_swe:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
pier_model_prefix: openai
|
|
pier_agent_kwargs:
|
|
model_class: litellm
|
|
researchrubrics:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
gdpval:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
tau3_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 16384
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
subset_list:
|
|
- banking_knowledge
|
|
extra_params:
|
|
user_model: deepseek-v4-pro
|
|
api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
|
|
api_base: https://api.deepseek.com/v1
|
|
generation_config:
|
|
temperature: 0.0
|
|
max_tokens: 4096
|
|
max_completion_tokens: 64000
|
|
toolathlon:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
server_host: 127.0.0.1
|
|
server_port: 8080
|
|
ws_proxy_port: 8081
|
|
timeout_seconds: 14400
|
|
submit_timeout_seconds: 300
|
|
task_list:
|
|
- paper-checker
|
|
skip_container_restart: false
|
|
override_output_dir: true
|
|
terminal_bench_v2_1:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
timeout_multiplier: 2.0
|
|
max_turns: 500
|
|
browsecomp:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
mcp_atlas:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
officeqa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
subset_list:
|
|
- officeqa_pro
|
|
deepsearchqa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
job_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
automation_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
toolset: api
|
|
mmmu_pro:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
max_image_bytes: 5mb
|
|
extra_params:
|
|
dataset_format: standard (4 options)
|
|
charxiv:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
subset_list:
|
|
- reasoning
|
|
math_vision:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
baby_vision:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
zerobench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
max_image_bytes: 10mb
|
|
world_vqa:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
omni_doc_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|
|
dataset_args:
|
|
extra_params:
|
|
match_method: quick_match
|
|
perception_bench:
|
|
generation_config:
|
|
temperature: 1.0
|
|
top_p: 1.0
|
|
stream: true
|
|
max_tokens: 32768
|
|
max_completion_tokens: 64000
|