From acb94e3e2043b9c0edd2ae92c47644d934ad07ab Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 14 Sep 2026 07:39:55 +0000 Subject: [PATCH] humaneval/gpqa_diamond: repeats 3 (align with the es reference runs) syy's es runs: humaneval x3, gpqa x2, aime25/26 x12 (already aligned), mmlu_pro/longbench_v2 x1 (temp=0 deterministic -- repeats are noise, and 3x 12k samples is pure cost). temp=1 benches get 3. Co-Authored-By: Claude --- evalharness/config/default.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evalharness/config/default.yaml b/evalharness/config/default.yaml index 1620fbc..219a3f4 100644 --- a/evalharness/config/default.yaml +++ b/evalharness/config/default.yaml @@ -23,6 +23,7 @@ imo_answerbench: temperature: 1.0 gpqa_diamond: temperature: 1.0 + repeats: 3 max_tokens: 8192 mmlu: max_tokens: 8192 @@ -42,6 +43,7 @@ trivia_qa: max_tokens: 8192 humaneval: temperature: 1.0 + repeats: 3 live_code_bench: temperature: 1.0 longbench_v2: