Config directory: dp4-nothink/ with 28 per-bench YAMLs (aime repeats=12, judged, agent, long-context) + README

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/config/README.md

@@ -0,0 +1,64 @@
+# 配置目录
+
+每个模型/协议一个文件夹，内含逐 bench 的 YAML 配置。
+
+```
+config/
+├── README.md              ← 本文件
+└── dp4-nothink/           ← 模型/协议名
+    ├── default.yaml       ← 协议级默认参数（所有 bench 继承）
+    ├── aime24.yaml        ← 逐 bench 覆盖
+    ├── aime25.yaml
+    ├── ...
+    └── swe_bench_verified.yaml
+```
+
+## 每个 YAML 的结构
+
+```yaml
+# generation: 传给模型的参数
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+# run: 运行方式
+run:
+  repeats: 12          # 跑 12 遍取均值（temp=1 方差测量用）
+  limit: null          # 全量 / limit_per_task: 10 每子集 10 条
+  concurrency: 4
+  checkpoint: true
+  resume: true
+
+# judge: LLM-judge 类 bench 需要
+judge:
+  model: dp4-flash
+  api_url: http://174.1.51.4:30000/v1
+```
+
+## 优先级
+
+```
+DatasetSpec.gen_config  <  default.yaml  <  <bench>.yaml  <  命令行显式参数
+```
+
+## 使用
+
+```bash
+# 单 bench
+evalharness eval run aime25 --config dp4-nothink --api-url ... --model ...
+
+# 多 bench（自动读各自的 yaml）
+evalharness eval run aime24 aime25 aime26 hmmt26 --config dp4-nothink ...
+
+# 查看某 bench 的生效配置
+evalharness config show dp4-nothink aime25
+```
+
+## 新增模型配置
+
+```bash
+mkdir config/qwen3-es-parity
+cp config/dp4-nothink/default.yaml config/qwen3-es-parity/
+# 编辑 default.yaml，然后按需添加逐 bench 覆盖
+```

evalharness/config/dp4-nothink/aime24.yaml

@@ -0,0 +1,10 @@
+# aime24: AIME/HMMT -- temp=1 sampling, 12 runs averaged
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  repeats: 12
+  limit: null          # full dataset (30 problems)
+  concurrency: 4

evalharness/config/dp4-nothink/aime25.yaml

@@ -0,0 +1,10 @@
+# aime25: AIME/HMMT -- temp=1 sampling, 12 runs averaged
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  repeats: 12
+  limit: null          # full dataset (30 problems)
+  concurrency: 4

evalharness/config/dp4-nothink/aime26.yaml

@@ -0,0 +1,10 @@
+# aime26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  repeats: 12
+  limit: null          # full dataset (30 problems)
+  concurrency: 4

evalharness/config/dp4-nothink/arc.yaml

@@ -0,0 +1,9 @@
+# arc: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/bbh.yaml

@@ -0,0 +1,9 @@
+# bbh: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/bfcl_v3.yaml

@@ -0,0 +1,13 @@
+generation:
+  temperature: 0.0
+  max_tokens: 4096
+  top_p: 1.0
+  parallel_tool_calls: true
+
+run:
+  limit_per_task: 20
+  concurrency: 8
+  env: bfcl_mock
+  extra:
+    is_fc_model: true
+    underscore_to_dot: true

evalharness/config/dp4-nothink/bigcodebench.yaml

@@ -0,0 +1,9 @@
+# bigcodebench: code generation (docker sandbox)
+generation:
+  temperature: 1.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 4

evalharness/config/dp4-nothink/cmmlu.yaml

@@ -0,0 +1,9 @@
+# cmmlu: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/default.yaml

@@ -0,0 +1,12 @@
+# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
+# All benches inherit these unless overridden in their own <bench>.yaml
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+  stream: true
+
+run:
+  concurrency: 8
+  checkpoint: true
+  resume: true

evalharness/config/dp4-nothink/drop.yaml

@@ -0,0 +1,9 @@
+# drop: short-answer QA
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 8

evalharness/config/dp4-nothink/general_fc.yaml

@@ -0,0 +1,8 @@
+generation:
+  temperature: 0.0
+  max_tokens: 4096
+  top_p: 1.0
+
+run:
+  limit: 400
+  concurrency: 8

evalharness/config/dp4-nothink/gpqa_diamond.yaml

@@ -0,0 +1,9 @@
+# gpqa_diamond: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/gsm8k.yaml

@@ -0,0 +1,9 @@
+# gsm8k: short-answer QA
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 8

evalharness/config/dp4-nothink/hellaswag.yaml

@@ -0,0 +1,9 @@
+# hellaswag: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/hle.yaml

@@ -0,0 +1,12 @@
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 25
+  concurrency: 8
+
+judge:
+  model: dp4-flash
+  api_url: http://174.1.51.4:30000/v1

evalharness/config/dp4-nothink/hmmt26.yaml

@@ -0,0 +1,10 @@
+# hmmt26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  repeats: 12
+  limit: null          # full dataset (30 problems)
+  concurrency: 4

evalharness/config/dp4-nothink/humaneval.yaml

@@ -0,0 +1,9 @@
+# humaneval: code generation (docker sandbox)
+generation:
+  temperature: 1.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 4

evalharness/config/dp4-nothink/imo_answerbench.yaml

@@ -0,0 +1,12 @@
+generation:
+  temperature: 1.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  limit_per_task: 25
+  concurrency: 8
+
+judge:
+  model: dp4-flash
+  api_url: http://174.1.51.4:30000/v1

evalharness/config/dp4-nothink/live_code_bench.yaml

@@ -0,0 +1,9 @@
+# live_code_bench: code generation (docker sandbox)
+generation:
+  temperature: 1.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 4

evalharness/config/dp4-nothink/longbench_v2.yaml

@@ -0,0 +1,9 @@
+generation:
+  temperature: 0.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  limit_per_task: 66
+  concurrency: 4
+  max_input_tokens: 120000

evalharness/config/dp4-nothink/mmlu.yaml

@@ -0,0 +1,9 @@
+# mmlu: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/mmlu_pro.yaml

@@ -0,0 +1,9 @@
+# mmlu_pro: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/dp4-nothink/openai_mrcr.yaml

@@ -0,0 +1,9 @@
+generation:
+  temperature: 0.0
+  max_tokens: 8192
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 4
+  max_input_tokens: 120000

evalharness/config/dp4-nothink/simple_qa.yaml

@@ -0,0 +1,9 @@
+# simple_qa: short-answer QA
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 8

evalharness/config/dp4-nothink/swe_bench_verified.yaml

@@ -0,0 +1,8 @@
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 70
+  concurrency: 4

evalharness/config/dp4-nothink/tau2_bench.yaml

@@ -0,0 +1,9 @@
+generation:
+  temperature: 0.0
+  max_tokens: 16384
+  top_p: 1.0
+
+run:
+  concurrency: 8
+  env: tau2_official
+  max_turns: 50

evalharness/config/dp4-nothink/trivia_qa.yaml

@@ -0,0 +1,9 @@
+# trivia_qa: short-answer QA
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit: 200
+  concurrency: 8

evalharness/config/dp4-nothink/winogrande.yaml

@@ -0,0 +1,9 @@
+# winogrande: knowledge/MCQ benchmark
+generation:
+  temperature: 0.0
+  max_tokens: 32768
+  top_p: 1.0
+
+run:
+  limit_per_task: 10
+  concurrency: 8

evalharness/config/gen_profiles.yaml

@@ -1,31 +0,0 @@
-# Generation-parameter profiles (YAML).
-# Usage: evalharness eval run <bench> --model ... --profile <name>
-# Resolution: DatasetSpec.gen_config < profile.default < profile.<bench> < gen_kwargs
-
-dp4-nothink:
-  default: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
-
-qwen3-es-parity:
-  default: {temperature: 0.0, max_tokens: 32768}
-  simple_qa: {max_tokens: 1024}
-  hle: {max_tokens: 8192}
-  gpqa_diamond: {temperature: 1.0, max_tokens: 8192}
-  aime24: {temperature: 1.0, max_tokens: 8192}
-  aime25: {temperature: 1.0, max_tokens: 8192}
-  aime26: {temperature: 1.0, max_tokens: 8192}
-  hmmt26: {temperature: 1.0, max_tokens: 8192}
-  imo_answerbench: {temperature: 1.0, max_tokens: 8192}
-
-glm53-nothink:
-  default: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
-  aime24: {temperature: 1.0, max_tokens: 8192}
-  aime25: {temperature: 1.0, max_tokens: 8192}
-  aime26: {temperature: 1.0, max_tokens: 8192}
-  hmmt26: {temperature: 1.0, max_tokens: 8192}
-  imo_answerbench: {temperature: 1.0, max_tokens: 8192}
-  gpqa_diamond: {temperature: 1.0, max_tokens: 8192}
-  humaneval: {temperature: 1.0, max_tokens: 32768}
-  live_code_bench: {temperature: 1.0, max_tokens: 32768}
-
-t1-short:
-  default: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}