From e9b79a2a412a2d8ccc40cad24d0653bb848509e1 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 02:10:50 +0000 Subject: [PATCH] thinking.yaml: 64k budgets for the full-thinking ladder (es parity) Co-Authored-By: Claude --- evalharness/config/thinking.yaml | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 evalharness/config/thinking.yaml diff --git a/evalharness/config/thinking.yaml b/evalharness/config/thinking.yaml new file mode 100644 index 0000000..1a001fe --- /dev/null +++ b/evalharness/config/thinking.yaml @@ -0,0 +1,60 @@ +default: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 200336 +aime24: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +aime25: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +aime26: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +hmmt26: + temperature: 1.0 + repeats: 12 + max_tokens: 200336 +imo_answerbench: + temperature: 1.0 +gpqa_diamond: + temperature: 1.0 + repeats: 3 + max_tokens: 200336 +mmlu: + max_tokens: 200336 +mmlu_pro: + max_tokens: 200336 +cmmlu: + max_tokens: 200336 +arc: + max_tokens: 200336 +hellaswag: + max_tokens: 200336 +winogrande: + max_tokens: 200336 +simple_qa: + max_tokens: 200336 +trivia_qa: + max_tokens: 200336 +humaneval: + temperature: 1.0 + repeats: 3 +live_code_bench: + temperature: 1.0 +longbench_v2: + max_tokens: 200336 + max_input_tokens: 128000 +openai_mrcr: + max_tokens: 200336 + max_input_tokens: 128000 +bfcl_v3: + max_tokens: 4096 +general_fc: + max_tokens: 4096 +tau2_bench: + max_tokens: 16384