Config: single dp4-nothink.yaml with all 28 bench configs (evalscope-style)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora
parent
c27a22555d
commit
d16009d054
119
evalharness/config/dp4-nothink.yaml
Normal file
119
evalharness/config/dp4-nothink.yaml
Normal file
@ -0,0 +1,119 @@
|
||||
# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
|
||||
# Usage: evalharness eval run <bench> --config dp4-nothink --api-url ... --model ...
|
||||
# Priority: DatasetSpec.gen_config < default < <bench> < CLI args
|
||||
|
||||
default:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0, stream: true}
|
||||
run: {concurrency: 8, checkpoint: true, resume: true}
|
||||
|
||||
# ── Math (temp=1, 12 runs averaged) ──
|
||||
aime24:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {repeats: 12, concurrency: 4}
|
||||
|
||||
aime25:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {repeats: 12, concurrency: 4}
|
||||
|
||||
aime26:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {repeats: 12, concurrency: 4}
|
||||
|
||||
hmmt26:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {repeats: 12, concurrency: 4}
|
||||
|
||||
competition_math:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit_per_task: 40, concurrency: 8}
|
||||
|
||||
gsm8k:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
# ── Knowledge / MCQ ──
|
||||
mmlu:
|
||||
run: {limit_per_task: 4, concurrency: 8}
|
||||
|
||||
cmmlu:
|
||||
run: {limit_per_task: 3, concurrency: 8}
|
||||
|
||||
mmlu_pro:
|
||||
run: {limit_per_task: 20, concurrency: 8}
|
||||
|
||||
arc:
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
hellaswag:
|
||||
run: {limit: 400, concurrency: 8}
|
||||
|
||||
winogrande:
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
bbh:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit_per_task: 10, concurrency: 8}
|
||||
|
||||
gpqa_diamond:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
# ── QA ──
|
||||
simple_qa:
|
||||
generation: {temperature: 0.0, max_tokens: 1024, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
trivia_qa:
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
drop:
|
||||
run: {limit: 200, concurrency: 8}
|
||||
|
||||
# ── Judged ──
|
||||
hle:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit_per_task: 25, concurrency: 8}
|
||||
judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
|
||||
|
||||
imo_answerbench:
|
||||
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {limit_per_task: 25, concurrency: 8}
|
||||
judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
|
||||
|
||||
# ── Code (docker sandbox) ──
|
||||
humaneval:
|
||||
generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {concurrency: 4}
|
||||
|
||||
bigcodebench:
|
||||
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 4}
|
||||
|
||||
live_code_bench:
|
||||
generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 4}
|
||||
|
||||
# ── Long context ──
|
||||
longbench_v2:
|
||||
generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {limit_per_task: 66, concurrency: 4, max_input_tokens: 120000}
|
||||
|
||||
openai_mrcr:
|
||||
generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
|
||||
run: {limit: 200, concurrency: 4, max_input_tokens: 120000}
|
||||
|
||||
# ── Agent / tools ──
|
||||
bfcl_v3:
|
||||
generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0, parallel_tool_calls: true}
|
||||
run: {limit_per_task: 20, concurrency: 8, env: bfcl_mock}
|
||||
|
||||
general_fc:
|
||||
generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0}
|
||||
run: {limit: 400, concurrency: 8}
|
||||
|
||||
tau2_bench:
|
||||
generation: {temperature: 0.0, max_tokens: 16384, top_p: 1.0}
|
||||
run: {concurrency: 8, env: tau2_official, max_turns: 50}
|
||||
|
||||
swe_bench_verified:
|
||||
run: {limit: 70, concurrency: 4}
|
||||
@ -1,10 +0,0 @@
|
||||
# aime24: AIME/HMMT -- temp=1 sampling, 12 runs averaged
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
repeats: 12
|
||||
limit: null # full dataset (30 problems)
|
||||
concurrency: 4
|
||||
@ -1,10 +0,0 @@
|
||||
# aime25: AIME/HMMT -- temp=1 sampling, 12 runs averaged
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
repeats: 12
|
||||
limit: null # full dataset (30 problems)
|
||||
concurrency: 4
|
||||
@ -1,10 +0,0 @@
|
||||
# aime26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
repeats: 12
|
||||
limit: null # full dataset (30 problems)
|
||||
concurrency: 4
|
||||
@ -1,9 +0,0 @@
|
||||
# arc: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# bbh: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,13 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 4096
|
||||
top_p: 1.0
|
||||
parallel_tool_calls: true
|
||||
|
||||
run:
|
||||
limit_per_task: 20
|
||||
concurrency: 8
|
||||
env: bfcl_mock
|
||||
extra:
|
||||
is_fc_model: true
|
||||
underscore_to_dot: true
|
||||
@ -1,9 +0,0 @@
|
||||
# bigcodebench: code generation (docker sandbox)
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 4
|
||||
@ -1,9 +0,0 @@
|
||||
# cmmlu: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,12 +0,0 @@
|
||||
# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
|
||||
# All benches inherit these unless overridden in their own <bench>.yaml
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
stream: true
|
||||
|
||||
run:
|
||||
concurrency: 8
|
||||
checkpoint: true
|
||||
resume: true
|
||||
@ -1,9 +0,0 @@
|
||||
# drop: short-answer QA
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 8
|
||||
@ -1,8 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 4096
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 400
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# gpqa_diamond: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# gsm8k: short-answer QA
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# hellaswag: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,12 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 25
|
||||
concurrency: 8
|
||||
|
||||
judge:
|
||||
model: dp4-flash
|
||||
api_url: http://174.1.51.4:30000/v1
|
||||
@ -1,10 +0,0 @@
|
||||
# hmmt26: AIME/HMMT -- temp=1 sampling, 12 runs averaged
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
repeats: 12
|
||||
limit: null # full dataset (30 problems)
|
||||
concurrency: 4
|
||||
@ -1,9 +0,0 @@
|
||||
# humaneval: code generation (docker sandbox)
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 4
|
||||
@ -1,12 +0,0 @@
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 25
|
||||
concurrency: 8
|
||||
|
||||
judge:
|
||||
model: dp4-flash
|
||||
api_url: http://174.1.51.4:30000/v1
|
||||
@ -1,9 +0,0 @@
|
||||
# live_code_bench: code generation (docker sandbox)
|
||||
generation:
|
||||
temperature: 1.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 4
|
||||
@ -1,9 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 66
|
||||
concurrency: 4
|
||||
max_input_tokens: 120000
|
||||
@ -1,9 +0,0 @@
|
||||
# mmlu: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# mmlu_pro: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 8192
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 4
|
||||
max_input_tokens: 120000
|
||||
@ -1,9 +0,0 @@
|
||||
# simple_qa: short-answer QA
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 8
|
||||
@ -1,8 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 70
|
||||
concurrency: 4
|
||||
@ -1,9 +0,0 @@
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 16384
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
concurrency: 8
|
||||
env: tau2_official
|
||||
max_turns: 50
|
||||
@ -1,9 +0,0 @@
|
||||
# trivia_qa: short-answer QA
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit: 200
|
||||
concurrency: 8
|
||||
@ -1,9 +0,0 @@
|
||||
# winogrande: knowledge/MCQ benchmark
|
||||
generation:
|
||||
temperature: 0.0
|
||||
max_tokens: 32768
|
||||
top_p: 1.0
|
||||
|
||||
run:
|
||||
limit_per_task: 10
|
||||
concurrency: 8
|
||||
Loading…
x
Reference in New Issue
Block a user