From c27a22555d43b2f738b017176aa5019b63569877 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 11 Sep 2026 07:59:42 +0000 Subject: [PATCH] Config directory: dp4-nothink/ with 28 per-bench YAMLs (aime repeats=12, judged, agent, long-context) + README Co-Authored-By: Claude --- evalharness/config/README.md | 64 +++++++++++++++++++ evalharness/config/dp4-nothink/aime24.yaml | 10 +++ evalharness/config/dp4-nothink/aime25.yaml | 10 +++ evalharness/config/dp4-nothink/aime26.yaml | 10 +++ evalharness/config/dp4-nothink/arc.yaml | 9 +++ evalharness/config/dp4-nothink/bbh.yaml | 9 +++ evalharness/config/dp4-nothink/bfcl_v3.yaml | 13 ++++ .../config/dp4-nothink/bigcodebench.yaml | 9 +++ evalharness/config/dp4-nothink/cmmlu.yaml | 9 +++ evalharness/config/dp4-nothink/default.yaml | 12 ++++ evalharness/config/dp4-nothink/drop.yaml | 9 +++ .../config/dp4-nothink/general_fc.yaml | 8 +++ .../config/dp4-nothink/gpqa_diamond.yaml | 9 +++ evalharness/config/dp4-nothink/gsm8k.yaml | 9 +++ evalharness/config/dp4-nothink/hellaswag.yaml | 9 +++ evalharness/config/dp4-nothink/hle.yaml | 12 ++++ evalharness/config/dp4-nothink/hmmt26.yaml | 10 +++ evalharness/config/dp4-nothink/humaneval.yaml | 9 +++ .../config/dp4-nothink/imo_answerbench.yaml | 12 ++++ .../config/dp4-nothink/live_code_bench.yaml | 9 +++ .../config/dp4-nothink/longbench_v2.yaml | 9 +++ evalharness/config/dp4-nothink/mmlu.yaml | 9 +++ evalharness/config/dp4-nothink/mmlu_pro.yaml | 9 +++ .../config/dp4-nothink/openai_mrcr.yaml | 9 +++ evalharness/config/dp4-nothink/simple_qa.yaml | 9 +++ .../dp4-nothink/swe_bench_verified.yaml | 8 +++ .../config/dp4-nothink/tau2_bench.yaml | 9 +++ evalharness/config/dp4-nothink/trivia_qa.yaml | 9 +++ .../config/dp4-nothink/winogrande.yaml | 9 +++ evalharness/config/gen_profiles.yaml | 31 --------- 30 files changed, 331 insertions(+), 31 deletions(-) create mode 100644 evalharness/config/README.md create mode 100644 evalharness/config/dp4-nothink/aime24.yaml create mode 100644 evalharness/config/dp4-nothink/aime25.yaml create mode 100644 evalharness/config/dp4-nothink/aime26.yaml create mode 100644 evalharness/config/dp4-nothink/arc.yaml create mode 100644 evalharness/config/dp4-nothink/bbh.yaml create mode 100644 evalharness/config/dp4-nothink/bfcl_v3.yaml create mode 100644 evalharness/config/dp4-nothink/bigcodebench.yaml create mode 100644 evalharness/config/dp4-nothink/cmmlu.yaml create mode 100644 evalharness/config/dp4-nothink/default.yaml create mode 100644 evalharness/config/dp4-nothink/drop.yaml create mode 100644 evalharness/config/dp4-nothink/general_fc.yaml create mode 100644 evalharness/config/dp4-nothink/gpqa_diamond.yaml create mode 100644 evalharness/config/dp4-nothink/gsm8k.yaml create mode 100644 evalharness/config/dp4-nothink/hellaswag.yaml create mode 100644 evalharness/config/dp4-nothink/hle.yaml create mode 100644 evalharness/config/dp4-nothink/hmmt26.yaml create mode 100644 evalharness/config/dp4-nothink/humaneval.yaml create mode 100644 evalharness/config/dp4-nothink/imo_answerbench.yaml create mode 100644 evalharness/config/dp4-nothink/live_code_bench.yaml create mode 100644 evalharness/config/dp4-nothink/longbench_v2.yaml create mode 100644 evalharness/config/dp4-nothink/mmlu.yaml create mode 100644 evalharness/config/dp4-nothink/mmlu_pro.yaml create mode 100644 evalharness/config/dp4-nothink/openai_mrcr.yaml create mode 100644 evalharness/config/dp4-nothink/simple_qa.yaml create mode 100644 evalharness/config/dp4-nothink/swe_bench_verified.yaml create mode 100644 evalharness/config/dp4-nothink/tau2_bench.yaml create mode 100644 evalharness/config/dp4-nothink/trivia_qa.yaml create mode 100644 evalharness/config/dp4-nothink/winogrande.yaml delete mode 100644 evalharness/config/gen_profiles.yaml diff --git a/evalharness/config/README.md b/evalharness/config/README.md new file mode 100644 index 0000000..e3c02a6 --- /dev/null +++ b/evalharness/config/README.md @@ -0,0 +1,64 @@ +# 配置目录 + +每个模型/协议一个文件夹，内含逐 bench 的 YAML 配置。 + +``` +config/ +├── README.md ← 本文件 +└── dp4-nothink/ ← 模型/协议名 + ├── default.yaml ← 协议级默认参数（所有 bench 继承） + ├── aime24.yaml ← 逐 bench 覆盖 + ├── aime25.yaml + ├── ... + └── swe_bench_verified.yaml +``` + +## 每个 YAML 的结构 + +```yaml +# generation: 传给模型的参数 +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +# run: 运行方式 +run: + repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用） + limit: null # 全量 / limit_per_task: 10 每子集 10 条 + concurrency: 4 + checkpoint: true + resume: true + +# judge: LLM-judge 类 bench 需要 +judge: + model: dp4-flash + api_url: http://174.1.51.4:30000/v1 +``` + +## 优先级 + +``` +DatasetSpec.gen_config < default.yaml < .yaml < 命令行显式参数 +``` + +## 使用 + +```bash +# 单 bench +evalharness eval run aime25 --config dp4-nothink --api-url ... --model ... + +# 多 bench（自动读各自的 yaml） +evalharness eval run aime24 aime25 aime26 hmmt26 --config dp4-nothink ... + +# 查看某 bench 的生效配置 +evalharness config show dp4-nothink aime25 +``` + +## 新增模型配置 + +```bash +mkdir config/qwen3-es-parity +cp config/dp4-nothink/default.yaml config/qwen3-es-parity/ +# 编辑 default.yaml，然后按需添加逐 bench 覆盖 +``` diff --git a/evalharness/config/dp4-nothink/aime24.yaml b/evalharness/config/dp4-nothink/aime24.yaml new file mode 100644 index 0000000..7f84a58 --- /dev/null +++ b/evalharness/config/dp4-nothink/aime24.yaml @@ -0,0 +1,10 @@ +# aime24: AIME/HMMT -- temp=1 sampling, 12 runs averaged +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +run: + repeats: 12 + limit: null # full dataset (30 problems) + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/aime25.yaml b/evalharness/config/dp4-nothink/aime25.yaml new file mode 100644 index 0000000..2be49f6 --- /dev/null +++ b/evalharness/config/dp4-nothink/aime25.yaml @@ -0,0 +1,10 @@ +# aime25: AIME/HMMT -- temp=1 sampling, 12 runs averaged +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +run: + repeats: 12 + limit: null # full dataset (30 problems) + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/aime26.yaml b/evalharness/config/dp4-nothink/aime26.yaml new file mode 100644 index 0000000..07bfc0d --- /dev/null +++ b/evalharness/config/dp4-nothink/aime26.yaml @@ -0,0 +1,10 @@ +# aime26: AIME/HMMT -- temp=1 sampling, 12 runs averaged +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +run: + repeats: 12 + limit: null # full dataset (30 problems) + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/arc.yaml b/evalharness/config/dp4-nothink/arc.yaml new file mode 100644 index 0000000..6421c5f --- /dev/null +++ b/evalharness/config/dp4-nothink/arc.yaml @@ -0,0 +1,9 @@ +# arc: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/bbh.yaml b/evalharness/config/dp4-nothink/bbh.yaml new file mode 100644 index 0000000..f6d6c88 --- /dev/null +++ b/evalharness/config/dp4-nothink/bbh.yaml @@ -0,0 +1,9 @@ +# bbh: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/bfcl_v3.yaml b/evalharness/config/dp4-nothink/bfcl_v3.yaml new file mode 100644 index 0000000..bb1b9ae --- /dev/null +++ b/evalharness/config/dp4-nothink/bfcl_v3.yaml @@ -0,0 +1,13 @@ +generation: + temperature: 0.0 + max_tokens: 4096 + top_p: 1.0 + parallel_tool_calls: true + +run: + limit_per_task: 20 + concurrency: 8 + env: bfcl_mock + extra: + is_fc_model: true + underscore_to_dot: true diff --git a/evalharness/config/dp4-nothink/bigcodebench.yaml b/evalharness/config/dp4-nothink/bigcodebench.yaml new file mode 100644 index 0000000..898e57c --- /dev/null +++ b/evalharness/config/dp4-nothink/bigcodebench.yaml @@ -0,0 +1,9 @@ +# bigcodebench: code generation (docker sandbox) +generation: + temperature: 1.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/cmmlu.yaml b/evalharness/config/dp4-nothink/cmmlu.yaml new file mode 100644 index 0000000..f0ea060 --- /dev/null +++ b/evalharness/config/dp4-nothink/cmmlu.yaml @@ -0,0 +1,9 @@ +# cmmlu: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/default.yaml b/evalharness/config/dp4-nothink/default.yaml new file mode 100644 index 0000000..6fe15cd --- /dev/null +++ b/evalharness/config/dp4-nothink/default.yaml @@ -0,0 +1,12 @@ +# dp4-nothink: DeepSeek-V4-Flash nothinking protocol +# All benches inherit these unless overridden in their own .yaml +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + stream: true + +run: + concurrency: 8 + checkpoint: true + resume: true diff --git a/evalharness/config/dp4-nothink/drop.yaml b/evalharness/config/dp4-nothink/drop.yaml new file mode 100644 index 0000000..63c5755 --- /dev/null +++ b/evalharness/config/dp4-nothink/drop.yaml @@ -0,0 +1,9 @@ +# drop: short-answer QA +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/general_fc.yaml b/evalharness/config/dp4-nothink/general_fc.yaml new file mode 100644 index 0000000..f896ba0 --- /dev/null +++ b/evalharness/config/dp4-nothink/general_fc.yaml @@ -0,0 +1,8 @@ +generation: + temperature: 0.0 + max_tokens: 4096 + top_p: 1.0 + +run: + limit: 400 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/gpqa_diamond.yaml b/evalharness/config/dp4-nothink/gpqa_diamond.yaml new file mode 100644 index 0000000..6dd39e9 --- /dev/null +++ b/evalharness/config/dp4-nothink/gpqa_diamond.yaml @@ -0,0 +1,9 @@ +# gpqa_diamond: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/gsm8k.yaml b/evalharness/config/dp4-nothink/gsm8k.yaml new file mode 100644 index 0000000..396fbc6 --- /dev/null +++ b/evalharness/config/dp4-nothink/gsm8k.yaml @@ -0,0 +1,9 @@ +# gsm8k: short-answer QA +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/hellaswag.yaml b/evalharness/config/dp4-nothink/hellaswag.yaml new file mode 100644 index 0000000..d069bbc --- /dev/null +++ b/evalharness/config/dp4-nothink/hellaswag.yaml @@ -0,0 +1,9 @@ +# hellaswag: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/hle.yaml b/evalharness/config/dp4-nothink/hle.yaml new file mode 100644 index 0000000..92f6fed --- /dev/null +++ b/evalharness/config/dp4-nothink/hle.yaml @@ -0,0 +1,12 @@ +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 25 + concurrency: 8 + +judge: + model: dp4-flash + api_url: http://174.1.51.4:30000/v1 diff --git a/evalharness/config/dp4-nothink/hmmt26.yaml b/evalharness/config/dp4-nothink/hmmt26.yaml new file mode 100644 index 0000000..690e119 --- /dev/null +++ b/evalharness/config/dp4-nothink/hmmt26.yaml @@ -0,0 +1,10 @@ +# hmmt26: AIME/HMMT -- temp=1 sampling, 12 runs averaged +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +run: + repeats: 12 + limit: null # full dataset (30 problems) + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/humaneval.yaml b/evalharness/config/dp4-nothink/humaneval.yaml new file mode 100644 index 0000000..6f02be5 --- /dev/null +++ b/evalharness/config/dp4-nothink/humaneval.yaml @@ -0,0 +1,9 @@ +# humaneval: code generation (docker sandbox) +generation: + temperature: 1.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/imo_answerbench.yaml b/evalharness/config/dp4-nothink/imo_answerbench.yaml new file mode 100644 index 0000000..111f7c8 --- /dev/null +++ b/evalharness/config/dp4-nothink/imo_answerbench.yaml @@ -0,0 +1,12 @@ +generation: + temperature: 1.0 + max_tokens: 8192 + top_p: 1.0 + +run: + limit_per_task: 25 + concurrency: 8 + +judge: + model: dp4-flash + api_url: http://174.1.51.4:30000/v1 diff --git a/evalharness/config/dp4-nothink/live_code_bench.yaml b/evalharness/config/dp4-nothink/live_code_bench.yaml new file mode 100644 index 0000000..97a5470 --- /dev/null +++ b/evalharness/config/dp4-nothink/live_code_bench.yaml @@ -0,0 +1,9 @@ +# live_code_bench: code generation (docker sandbox) +generation: + temperature: 1.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/longbench_v2.yaml b/evalharness/config/dp4-nothink/longbench_v2.yaml new file mode 100644 index 0000000..4982493 --- /dev/null +++ b/evalharness/config/dp4-nothink/longbench_v2.yaml @@ -0,0 +1,9 @@ +generation: + temperature: 0.0 + max_tokens: 8192 + top_p: 1.0 + +run: + limit_per_task: 66 + concurrency: 4 + max_input_tokens: 120000 diff --git a/evalharness/config/dp4-nothink/mmlu.yaml b/evalharness/config/dp4-nothink/mmlu.yaml new file mode 100644 index 0000000..e6696fa --- /dev/null +++ b/evalharness/config/dp4-nothink/mmlu.yaml @@ -0,0 +1,9 @@ +# mmlu: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/mmlu_pro.yaml b/evalharness/config/dp4-nothink/mmlu_pro.yaml new file mode 100644 index 0000000..fe0528e --- /dev/null +++ b/evalharness/config/dp4-nothink/mmlu_pro.yaml @@ -0,0 +1,9 @@ +# mmlu_pro: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/openai_mrcr.yaml b/evalharness/config/dp4-nothink/openai_mrcr.yaml new file mode 100644 index 0000000..7eed969 --- /dev/null +++ b/evalharness/config/dp4-nothink/openai_mrcr.yaml @@ -0,0 +1,9 @@ +generation: + temperature: 0.0 + max_tokens: 8192 + top_p: 1.0 + +run: + limit: 200 + concurrency: 4 + max_input_tokens: 120000 diff --git a/evalharness/config/dp4-nothink/simple_qa.yaml b/evalharness/config/dp4-nothink/simple_qa.yaml new file mode 100644 index 0000000..af01b6c --- /dev/null +++ b/evalharness/config/dp4-nothink/simple_qa.yaml @@ -0,0 +1,9 @@ +# simple_qa: short-answer QA +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/swe_bench_verified.yaml b/evalharness/config/dp4-nothink/swe_bench_verified.yaml new file mode 100644 index 0000000..0077ec0 --- /dev/null +++ b/evalharness/config/dp4-nothink/swe_bench_verified.yaml @@ -0,0 +1,8 @@ +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 70 + concurrency: 4 diff --git a/evalharness/config/dp4-nothink/tau2_bench.yaml b/evalharness/config/dp4-nothink/tau2_bench.yaml new file mode 100644 index 0000000..64ea2ad --- /dev/null +++ b/evalharness/config/dp4-nothink/tau2_bench.yaml @@ -0,0 +1,9 @@ +generation: + temperature: 0.0 + max_tokens: 16384 + top_p: 1.0 + +run: + concurrency: 8 + env: tau2_official + max_turns: 50 diff --git a/evalharness/config/dp4-nothink/trivia_qa.yaml b/evalharness/config/dp4-nothink/trivia_qa.yaml new file mode 100644 index 0000000..5279c87 --- /dev/null +++ b/evalharness/config/dp4-nothink/trivia_qa.yaml @@ -0,0 +1,9 @@ +# trivia_qa: short-answer QA +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit: 200 + concurrency: 8 diff --git a/evalharness/config/dp4-nothink/winogrande.yaml b/evalharness/config/dp4-nothink/winogrande.yaml new file mode 100644 index 0000000..3e4515a --- /dev/null +++ b/evalharness/config/dp4-nothink/winogrande.yaml @@ -0,0 +1,9 @@ +# winogrande: knowledge/MCQ benchmark +generation: + temperature: 0.0 + max_tokens: 32768 + top_p: 1.0 + +run: + limit_per_task: 10 + concurrency: 8 diff --git a/evalharness/config/gen_profiles.yaml b/evalharness/config/gen_profiles.yaml deleted file mode 100644 index bb1fe27..0000000 --- a/evalharness/config/gen_profiles.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Generation-parameter profiles (YAML). -# Usage: evalharness eval run --model ... --profile -# Resolution: DatasetSpec.gen_config < profile.default < profile. < gen_kwargs - -dp4-nothink: - default: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} - -qwen3-es-parity: - default: {temperature: 0.0, max_tokens: 32768} - simple_qa: {max_tokens: 1024} - hle: {max_tokens: 8192} - gpqa_diamond: {temperature: 1.0, max_tokens: 8192} - aime24: {temperature: 1.0, max_tokens: 8192} - aime25: {temperature: 1.0, max_tokens: 8192} - aime26: {temperature: 1.0, max_tokens: 8192} - hmmt26: {temperature: 1.0, max_tokens: 8192} - imo_answerbench: {temperature: 1.0, max_tokens: 8192} - -glm53-nothink: - default: {temperature: 0.0, max_tokens: 8192, top_p: 1.0} - aime24: {temperature: 1.0, max_tokens: 8192} - aime25: {temperature: 1.0, max_tokens: 8192} - aime26: {temperature: 1.0, max_tokens: 8192} - hmmt26: {temperature: 1.0, max_tokens: 8192} - imo_answerbench: {temperature: 1.0, max_tokens: 8192} - gpqa_diamond: {temperature: 1.0, max_tokens: 8192} - humaneval: {temperature: 1.0, max_tokens: 32768} - live_code_bench: {temperature: 1.0, max_tokens: 32768} - -t1-short: - default: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}