Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Config simplified: default block + per-bench overrides only (removed env/limit/judge/concurrency; max_input_tokens 128000)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 08:42:21 +00:00
parent 0815f6e4f8
commit eb6028e8c9
1 changed files with 61 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

209
evalharness/config/dp4-nothink.yaml
View File

@ -1,196 +1,75 @@
gpqa_diamond:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
hle:
generation_config:
default:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
aime24:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
repeats: 12
aime25:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
repeats: 12
mmlu_pro:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
simple_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
arc:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
bbh:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
live_code_bench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
dataset_args:
subset_list:
- release_v6
aime26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
repeats: 12
hmmt26:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
repeats: 12
imo_answerbench:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
drop:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
hellaswag:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
gpqa_diamond:
temperature: 1.0
max_tokens: 8192
mmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
openai_mrcr:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
mmlu_pro:
max_tokens: 8192
bigcodebench:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
humaneval:
generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
gsm8k:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
competition_math:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
cmmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
trivia_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
arc:
max_tokens: 8192
hellaswag:
max_tokens: 8192
winogrande:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
simple_qa:
max_tokens: 8192
trivia_qa:
max_tokens: 8192
humaneval:
temperature: 1.0
live_code_bench:
temperature: 1.0
longbench_v2:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
dataset_args:
subset_list:
- short
- medium
- long
tau2_bench:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 16384
dataset_args:
extra_params:
user_model: deepseek-v4-pro
api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
api_base: https://api.deepseek.com/v1
generation_config:
temperature: 0.0
max_tokens: 4096
env: tau2_official
max_turns: 50
general_fc:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096
max_input_tokens: 128000
openai_mrcr:
max_tokens: 8192
max_input_tokens: 128000
bfcl_v3:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096
parallel_tool_calls: true
dataset_args:
extra_params:
is_fc_model: true
underscore_to_dot: true
env: bfcl_mock
swe_bench_verified:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
general_fc:
max_tokens: 4096
tau2_bench:
max_tokens: 16384