diff --git a/evalharness/config/thinking.yaml b/evalharness/config/thinking.yaml new file mode 100644 index 0000000..1a001fe --- /dev/null +++ b/evalharness/config/thinking.yaml @@ -0,0 +1,60 @@ +default: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 200336 +aime24: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +aime25: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +aime26: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +hmmt26: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +imo_answerbench: + temperature: 1.0 +gpqa_diamond: + temperature: 1.0 + repeats: 3 + max_tokens: 200336 +mmlu: + max_tokens: 200336 +mmlu_pro: + max_tokens: 200336 +cmmlu: + max_tokens: 200336 +arc: + max_tokens: 200336 +hellaswag: + max_tokens: 200336 +winogrande: + max_tokens: 200336 +simple_qa: + max_tokens: 200336 +trivia_qa: + max_tokens: 200336 +humaneval: + temperature: 1.0 + repeats: 3 +live_code_bench: + temperature: 1.0 +longbench_v2: + max_tokens: 200336 + max_input_tokens: 128000 +openai_mrcr: + max_tokens: 200336 + max_input_tokens: 128000 +bfcl_v3: + max_tokens: 4096 +general_fc: + max_tokens: 4096 +tau2_bench: + max_tokens: 16384