Config: single dp4-nothink.yaml with all 28 bench configs (evalscope-style)

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/config/dp4-nothink.yaml

@@ -0,0 +1,119 @@
+# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
+# Usage: evalharness eval run <bench> --config dp4-nothink --api-url ... --model ...
+# Priority: DatasetSpec.gen_config < default < <bench> < CLI args
+
+default:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0, stream: true}
+  run: {concurrency: 8, checkpoint: true, resume: true}
+
+# ── Math (temp=1, 12 runs averaged) ──
+aime24:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {repeats: 12, concurrency: 4}
+
+aime25:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {repeats: 12, concurrency: 4}
+
+aime26:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {repeats: 12, concurrency: 4}
+
+hmmt26:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {repeats: 12, concurrency: 4}
+
+competition_math:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit_per_task: 40, concurrency: 8}
+
+gsm8k:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit: 200, concurrency: 8}
+
+# ── Knowledge / MCQ ──
+mmlu:
+  run: {limit_per_task: 4, concurrency: 8}
+
+cmmlu:
+  run: {limit_per_task: 3, concurrency: 8}
+
+mmlu_pro:
+  run: {limit_per_task: 20, concurrency: 8}
+
+arc:
+  run: {limit: 200, concurrency: 8}
+
+hellaswag:
+  run: {limit: 400, concurrency: 8}
+
+winogrande:
+  run: {limit: 200, concurrency: 8}
+
+bbh:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit_per_task: 10, concurrency: 8}
+
+gpqa_diamond:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {limit: 200, concurrency: 8}
+
+# ── QA ──
+simple_qa:
+  generation: {temperature: 0.0, max_tokens: 1024, top_p: 1.0}
+  run: {limit: 200, concurrency: 8}
+
+trivia_qa:
+  run: {limit: 200, concurrency: 8}
+
+drop:
+  run: {limit: 200, concurrency: 8}
+
+# ── Judged ──
+hle:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit_per_task: 25, concurrency: 8}
+  judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
+
+imo_answerbench:
+  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
+  run: {limit_per_task: 25, concurrency: 8}
+  judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
+
+# ── Code (docker sandbox) ──
+humaneval:
+  generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
+  run: {concurrency: 4}
+
+bigcodebench:
+  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit: 200, concurrency: 4}
+
+live_code_bench:
+  generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
+  run: {limit: 200, concurrency: 4}
+
+# ── Long context ──
+longbench_v2:
+  generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
+  run: {limit_per_task: 66, concurrency: 4, max_input_tokens: 120000}
+
+openai_mrcr:
+  generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
+  run: {limit: 200, concurrency: 4, max_input_tokens: 120000}
+
+# ── Agent / tools ──
+bfcl_v3:
+  generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0, parallel_tool_calls: true}
+  run: {limit_per_task: 20, concurrency: 8, env: bfcl_mock}
+
+general_fc:
+  generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0}
+  run: {limit: 400, concurrency: 8}
+
+tau2_bench:
+  generation: {temperature: 0.0, max_tokens: 16384, top_p: 1.0}
+  run: {concurrency: 8, env: tau2_official, max_turns: 50}
+
+swe_bench_verified:
+  run: {limit: 70, concurrency: 4}

evalharness/config/dp4-nothink/aime24.yaml

@@ -1,10 +0,0 @@
-# aime24: AIME/HMMT -- temp=1 sampling, 12 runs averaged
-generation:
-  temperature: 1.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  repeats: 12
-  limit: null          # full dataset (30 problems)
-  concurrency: 4

evalharness/config/dp4-nothink/aime25.yaml

@@ -1,10 +0,0 @@
-# aime25: AIME/HMMT -- temp=1 sampling, 12 runs averaged
-generation:
-  temperature: 1.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  repeats: 12
-  limit: null          # full dataset (30 problems)
-  concurrency: 4

evalharness/config/dp4-nothink/aime26.yaml

@@ -1,10 +0,0 @@
-# aime26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
-generation:
-  temperature: 1.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  repeats: 12
-  limit: null          # full dataset (30 problems)
-  concurrency: 4

evalharness/config/dp4-nothink/arc.yaml

@@ -1,9 +0,0 @@
-# arc: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/bbh.yaml

@@ -1,9 +0,0 @@
-# bbh: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/bfcl_v3.yaml

@@ -1,13 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 4096
-  top_p: 1.0
-  parallel_tool_calls: true
-
-run:
-  limit_per_task: 20
-  concurrency: 8
-  env: bfcl_mock
-  extra:
-    is_fc_model: true
-    underscore_to_dot: true

evalharness/config/dp4-nothink/bigcodebench.yaml

@@ -1,9 +0,0 @@
-# bigcodebench: code generation (docker sandbox)
-generation:
-  temperature: 1.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 4

evalharness/config/dp4-nothink/cmmlu.yaml

@@ -1,9 +0,0 @@
-# cmmlu: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/default.yaml

@@ -1,12 +0,0 @@
-# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
-# All benches inherit these unless overridden in their own <bench>.yaml
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-  stream: true
-
-run:
-  concurrency: 8
-  checkpoint: true
-  resume: true

evalharness/config/dp4-nothink/drop.yaml

@@ -1,9 +0,0 @@
-# drop: short-answer QA
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 8

evalharness/config/dp4-nothink/general_fc.yaml

@@ -1,8 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 4096
-  top_p: 1.0
-
-run:
-  limit: 400
-  concurrency: 8

evalharness/config/dp4-nothink/gpqa_diamond.yaml

@@ -1,9 +0,0 @@
-# gpqa_diamond: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/gsm8k.yaml

@@ -1,9 +0,0 @@
-# gsm8k: short-answer QA
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 8

evalharness/config/dp4-nothink/hellaswag.yaml

@@ -1,9 +0,0 @@
-# hellaswag: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/hle.yaml

@@ -1,12 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 25
-  concurrency: 8
-
-judge:
-  model: dp4-flash
-  api_url: http://174.1.51.4:30000/v1

evalharness/config/dp4-nothink/hmmt26.yaml

@@ -1,10 +0,0 @@
-# hmmt26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
-generation:
-  temperature: 1.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  repeats: 12
-  limit: null          # full dataset (30 problems)
-  concurrency: 4

evalharness/config/dp4-nothink/humaneval.yaml

@@ -1,9 +0,0 @@
-# humaneval: code generation (docker sandbox)
-generation:
-  temperature: 1.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 4

evalharness/config/dp4-nothink/imo_answerbench.yaml

@@ -1,12 +0,0 @@
-generation:
-  temperature: 1.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  limit_per_task: 25
-  concurrency: 8
-
-judge:
-  model: dp4-flash
-  api_url: http://174.1.51.4:30000/v1

evalharness/config/dp4-nothink/live_code_bench.yaml

@@ -1,9 +0,0 @@
-# live_code_bench: code generation (docker sandbox)
-generation:
-  temperature: 1.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 4

evalharness/config/dp4-nothink/longbench_v2.yaml

@@ -1,9 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  limit_per_task: 66
-  concurrency: 4
-  max_input_tokens: 120000

evalharness/config/dp4-nothink/mmlu.yaml

@@ -1,9 +0,0 @@
-# mmlu: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/mmlu_pro.yaml

@@ -1,9 +0,0 @@
-# mmlu_pro: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8

evalharness/config/dp4-nothink/openai_mrcr.yaml

@@ -1,9 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 8192
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 4
-  max_input_tokens: 120000

evalharness/config/dp4-nothink/simple_qa.yaml

@@ -1,9 +0,0 @@
-# simple_qa: short-answer QA
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 8

evalharness/config/dp4-nothink/swe_bench_verified.yaml

@@ -1,8 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 70
-  concurrency: 4

evalharness/config/dp4-nothink/tau2_bench.yaml

@@ -1,9 +0,0 @@
-generation:
-  temperature: 0.0
-  max_tokens: 16384
-  top_p: 1.0
-
-run:
-  concurrency: 8
-  env: tau2_official
-  max_turns: 50

evalharness/config/dp4-nothink/trivia_qa.yaml

@@ -1,9 +0,0 @@
-# trivia_qa: short-answer QA
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit: 200
-  concurrency: 8

evalharness/config/dp4-nothink/winogrande.yaml

@@ -1,9 +0,0 @@
-# winogrande: knowledge/MCQ benchmark
-generation:
-  temperature: 0.0
-  max_tokens: 32768
-  top_p: 1.0
-
-run:
-  limit_per_task: 10
-  concurrency: 8