diff --git a/evalharness/config/dp4-nothink.yaml b/evalharness/config/dp4-nothink.yaml new file mode 100644 index 0000000..19caaad --- /dev/null +++ b/evalharness/config/dp4-nothink.yaml @@ -0,0 +1,119 @@ +# dp4-nothink: DeepSeek-V4-Flash nothinking protocol +# Usage: evalharness eval run --config dp4-nothink --api-url ... --model ... +# Priority: DatasetSpec.gen_config < default < < CLI args + +default: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0, stream: true} + run: {concurrency: 8, checkpoint: true, resume: true} + +# ── Math (temp=1, 12 runs averaged) ── +aime24: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {repeats: 12, concurrency: 4} + +aime25: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {repeats: 12, concurrency: 4} + +aime26: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {repeats: 12, concurrency: 4} + +hmmt26: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {repeats: 12, concurrency: 4} + +competition_math: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + run: {limit_per_task: 40, concurrency: 8} + +gsm8k: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + run: {limit: 200, concurrency: 8} + +# ── Knowledge / MCQ ── +mmlu: + run: {limit_per_task: 4, concurrency: 8} + +cmmlu: + run: {limit_per_task: 3, concurrency: 8} + +mmlu_pro: + run: {limit_per_task: 20, concurrency: 8} + +arc: + run: {limit: 200, concurrency: 8} + +hellaswag: + run: {limit: 400, concurrency: 8} + +winogrande: + run: {limit: 200, concurrency: 8} + +bbh: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + run: {limit_per_task: 10, concurrency: 8} + +gpqa_diamond: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {limit: 200, concurrency: 8} + +# ── QA ── +simple_qa: + generation: {temperature: 0.0, max_tokens: 1024, top_p: 1.0} + run: {limit: 200, concurrency: 8} + +trivia_qa: + run: {limit: 200, concurrency: 8} + +drop: + run: {limit: 200, concurrency: 8} + +# ── Judged ── +hle: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + run: {limit_per_task: 25, concurrency: 8} + judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'} + +imo_answerbench: + generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} + run: {limit_per_task: 25, concurrency: 8} + judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'} + +# ── Code (docker sandbox) ── +humaneval: + generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0} + run: {concurrency: 4} + +bigcodebench: + generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + run: {limit: 200, concurrency: 4} + +live_code_bench: + generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0} + run: {limit: 200, concurrency: 4} + +# ── Long context ── +longbench_v2: + generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0} + run: {limit_per_task: 66, concurrency: 4, max_input_tokens: 120000} + +openai_mrcr: + generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0} + run: {limit: 200, concurrency: 4, max_input_tokens: 120000} + +# ── Agent / tools ── +bfcl_v3: + generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0, parallel_tool_calls: true} + run: {limit_per_task: 20, concurrency: 8, env: bfcl_mock} + +general_fc: + generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0} + run: {limit: 400, concurrency: 8} + +tau2_bench: + generation: {temperature: 0.0, max_tokens: 16384, top_p: 1.0} + run: {concurrency: 8, env: tau2_official, max_turns: 50} + +swe_bench_verified: + run: {limit: 70, concurrency: 4} diff --git a/evalharness/config/dp4-nothink/aime24.yaml b/evalharness/config/dp4-nothink/aime24.yaml deleted file mode 100644 index 7f84a58..0000000 --- a/evalharness/config/dp4-nothink/aime24.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# aime24: AIME/HMMT -- temp=1 sampling, 12 runs averaged -generation: - temperature: 1.0 - max_tokens: 8192 - top_p: 1.0 - -run: - repeats: 12 - limit: null # full dataset (30 problems) - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/aime25.yaml b/evalharness/config/dp4-nothink/aime25.yaml deleted file mode 100644 index 2be49f6..0000000 --- a/evalharness/config/dp4-nothink/aime25.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# aime25: AIME/HMMT -- temp=1 sampling, 12 runs averaged -generation: - temperature: 1.0 - max_tokens: 8192 - top_p: 1.0 - -run: - repeats: 12 - limit: null # full dataset (30 problems) - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/aime26.yaml b/evalharness/config/dp4-nothink/aime26.yaml deleted file mode 100644 index 07bfc0d..0000000 --- a/evalharness/config/dp4-nothink/aime26.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# aime26: AIME/HMMT -- temp=1 sampling, 12 runs averaged -generation: - temperature: 1.0 - max_tokens: 8192 - top_p: 1.0 - -run: - repeats: 12 - limit: null # full dataset (30 problems) - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/arc.yaml b/evalharness/config/dp4-nothink/arc.yaml deleted file mode 100644 index 6421c5f..0000000 --- a/evalharness/config/dp4-nothink/arc.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# arc: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/bbh.yaml b/evalharness/config/dp4-nothink/bbh.yaml deleted file mode 100644 index f6d6c88..0000000 --- a/evalharness/config/dp4-nothink/bbh.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# bbh: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/bfcl_v3.yaml b/evalharness/config/dp4-nothink/bfcl_v3.yaml deleted file mode 100644 index bb1b9ae..0000000 --- a/evalharness/config/dp4-nothink/bfcl_v3.yaml +++ /dev/null @@ -1,13 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 4096 - top_p: 1.0 - parallel_tool_calls: true - -run: - limit_per_task: 20 - concurrency: 8 - env: bfcl_mock - extra: - is_fc_model: true - underscore_to_dot: true diff --git a/evalharness/config/dp4-nothink/bigcodebench.yaml b/evalharness/config/dp4-nothink/bigcodebench.yaml deleted file mode 100644 index 898e57c..0000000 --- a/evalharness/config/dp4-nothink/bigcodebench.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# bigcodebench: code generation (docker sandbox) -generation: - temperature: 1.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/cmmlu.yaml b/evalharness/config/dp4-nothink/cmmlu.yaml deleted file mode 100644 index f0ea060..0000000 --- a/evalharness/config/dp4-nothink/cmmlu.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# cmmlu: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/default.yaml b/evalharness/config/dp4-nothink/default.yaml deleted file mode 100644 index 6fe15cd..0000000 --- a/evalharness/config/dp4-nothink/default.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# dp4-nothink: DeepSeek-V4-Flash nothinking protocol -# All benches inherit these unless overridden in their own .yaml -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - stream: true - -run: - concurrency: 8 - checkpoint: true - resume: true diff --git a/evalharness/config/dp4-nothink/drop.yaml b/evalharness/config/dp4-nothink/drop.yaml deleted file mode 100644 index 63c5755..0000000 --- a/evalharness/config/dp4-nothink/drop.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# drop: short-answer QA -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/general_fc.yaml b/evalharness/config/dp4-nothink/general_fc.yaml deleted file mode 100644 index f896ba0..0000000 --- a/evalharness/config/dp4-nothink/general_fc.yaml +++ /dev/null @@ -1,8 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 4096 - top_p: 1.0 - -run: - limit: 400 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/gpqa_diamond.yaml b/evalharness/config/dp4-nothink/gpqa_diamond.yaml deleted file mode 100644 index 6dd39e9..0000000 --- a/evalharness/config/dp4-nothink/gpqa_diamond.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# gpqa_diamond: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/gsm8k.yaml b/evalharness/config/dp4-nothink/gsm8k.yaml deleted file mode 100644 index 396fbc6..0000000 --- a/evalharness/config/dp4-nothink/gsm8k.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# gsm8k: short-answer QA -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/hellaswag.yaml b/evalharness/config/dp4-nothink/hellaswag.yaml deleted file mode 100644 index d069bbc..0000000 --- a/evalharness/config/dp4-nothink/hellaswag.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# hellaswag: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/hle.yaml b/evalharness/config/dp4-nothink/hle.yaml deleted file mode 100644 index 92f6fed..0000000 --- a/evalharness/config/dp4-nothink/hle.yaml +++ /dev/null @@ -1,12 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 25 - concurrency: 8 - -judge: - model: dp4-flash - api_url: http://174.1.51.4:30000/v1 diff --git a/evalharness/config/dp4-nothink/hmmt26.yaml b/evalharness/config/dp4-nothink/hmmt26.yaml deleted file mode 100644 index 690e119..0000000 --- a/evalharness/config/dp4-nothink/hmmt26.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# hmmt26: AIME/HMMT -- temp=1 sampling, 12 runs averaged -generation: - temperature: 1.0 - max_tokens: 8192 - top_p: 1.0 - -run: - repeats: 12 - limit: null # full dataset (30 problems) - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/humaneval.yaml b/evalharness/config/dp4-nothink/humaneval.yaml deleted file mode 100644 index 6f02be5..0000000 --- a/evalharness/config/dp4-nothink/humaneval.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# humaneval: code generation (docker sandbox) -generation: - temperature: 1.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/imo_answerbench.yaml b/evalharness/config/dp4-nothink/imo_answerbench.yaml deleted file mode 100644 index 111f7c8..0000000 --- a/evalharness/config/dp4-nothink/imo_answerbench.yaml +++ /dev/null @@ -1,12 +0,0 @@ -generation: - temperature: 1.0 - max_tokens: 8192 - top_p: 1.0 - -run: - limit_per_task: 25 - concurrency: 8 - -judge: - model: dp4-flash - api_url: http://174.1.51.4:30000/v1 diff --git a/evalharness/config/dp4-nothink/live_code_bench.yaml b/evalharness/config/dp4-nothink/live_code_bench.yaml deleted file mode 100644 index 97a5470..0000000 --- a/evalharness/config/dp4-nothink/live_code_bench.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# live_code_bench: code generation (docker sandbox) -generation: - temperature: 1.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/longbench_v2.yaml b/evalharness/config/dp4-nothink/longbench_v2.yaml deleted file mode 100644 index 4982493..0000000 --- a/evalharness/config/dp4-nothink/longbench_v2.yaml +++ /dev/null @@ -1,9 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 8192 - top_p: 1.0 - -run: - limit_per_task: 66 - concurrency: 4 - max_input_tokens: 120000 diff --git a/evalharness/config/dp4-nothink/mmlu.yaml b/evalharness/config/dp4-nothink/mmlu.yaml deleted file mode 100644 index e6696fa..0000000 --- a/evalharness/config/dp4-nothink/mmlu.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# mmlu: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/mmlu_pro.yaml b/evalharness/config/dp4-nothink/mmlu_pro.yaml deleted file mode 100644 index fe0528e..0000000 --- a/evalharness/config/dp4-nothink/mmlu_pro.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# mmlu_pro: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/openai_mrcr.yaml b/evalharness/config/dp4-nothink/openai_mrcr.yaml deleted file mode 100644 index 7eed969..0000000 --- a/evalharness/config/dp4-nothink/openai_mrcr.yaml +++ /dev/null @@ -1,9 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 8192 - top_p: 1.0 - -run: - limit: 200 - concurrency: 4 - max_input_tokens: 120000 diff --git a/evalharness/config/dp4-nothink/simple_qa.yaml b/evalharness/config/dp4-nothink/simple_qa.yaml deleted file mode 100644 index af01b6c..0000000 --- a/evalharness/config/dp4-nothink/simple_qa.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# simple_qa: short-answer QA -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/swe_bench_verified.yaml b/evalharness/config/dp4-nothink/swe_bench_verified.yaml deleted file mode 100644 index 0077ec0..0000000 --- a/evalharness/config/dp4-nothink/swe_bench_verified.yaml +++ /dev/null @@ -1,8 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 70 - concurrency: 4 diff --git a/evalharness/config/dp4-nothink/tau2_bench.yaml b/evalharness/config/dp4-nothink/tau2_bench.yaml deleted file mode 100644 index 64ea2ad..0000000 --- a/evalharness/config/dp4-nothink/tau2_bench.yaml +++ /dev/null @@ -1,9 +0,0 @@ -generation: - temperature: 0.0 - max_tokens: 16384 - top_p: 1.0 - -run: - concurrency: 8 - env: tau2_official - max_turns: 50 diff --git a/evalharness/config/dp4-nothink/trivia_qa.yaml b/evalharness/config/dp4-nothink/trivia_qa.yaml deleted file mode 100644 index 5279c87..0000000 --- a/evalharness/config/dp4-nothink/trivia_qa.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# trivia_qa: short-answer QA -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit: 200 - concurrency: 8 diff --git a/evalharness/config/dp4-nothink/winogrande.yaml b/evalharness/config/dp4-nothink/winogrande.yaml deleted file mode 100644 index 3e4515a..0000000 --- a/evalharness/config/dp4-nothink/winogrande.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# winogrande: knowledge/MCQ benchmark -generation: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 - -run: - limit_per_task: 10 - concurrency: 8