120 lines
3.3 KiB
YAML
120 lines
3.3 KiB
YAML
# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
|
|
# Usage: evalharness eval run <bench> --config dp4-nothink --api-url ... --model ...
|
|
# Priority: DatasetSpec.gen_config < default < <bench> < CLI args
|
|
|
|
default:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0, stream: true}
|
|
run: {concurrency: 8, checkpoint: true, resume: true}
|
|
|
|
# ── Math (temp=1, 12 runs averaged) ──
|
|
aime24:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {repeats: 12, concurrency: 4}
|
|
|
|
aime25:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {repeats: 12, concurrency: 4}
|
|
|
|
aime26:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {repeats: 12, concurrency: 4}
|
|
|
|
hmmt26:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {repeats: 12, concurrency: 4}
|
|
|
|
competition_math:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit_per_task: 40, concurrency: 8}
|
|
|
|
gsm8k:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
# ── Knowledge / MCQ ──
|
|
mmlu:
|
|
run: {limit_per_task: 4, concurrency: 8}
|
|
|
|
cmmlu:
|
|
run: {limit_per_task: 3, concurrency: 8}
|
|
|
|
mmlu_pro:
|
|
run: {limit_per_task: 20, concurrency: 8}
|
|
|
|
arc:
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
hellaswag:
|
|
run: {limit: 400, concurrency: 8}
|
|
|
|
winogrande:
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
bbh:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit_per_task: 10, concurrency: 8}
|
|
|
|
gpqa_diamond:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
# ── QA ──
|
|
simple_qa:
|
|
generation: {temperature: 0.0, max_tokens: 1024, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
trivia_qa:
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
drop:
|
|
run: {limit: 200, concurrency: 8}
|
|
|
|
# ── Judged ──
|
|
hle:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit_per_task: 25, concurrency: 8}
|
|
judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
|
|
|
|
imo_answerbench:
|
|
generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {limit_per_task: 25, concurrency: 8}
|
|
judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}
|
|
|
|
# ── Code (docker sandbox) ──
|
|
humaneval:
|
|
generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {concurrency: 4}
|
|
|
|
bigcodebench:
|
|
generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 4}
|
|
|
|
live_code_bench:
|
|
generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 4}
|
|
|
|
# ── Long context ──
|
|
longbench_v2:
|
|
generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {limit_per_task: 66, concurrency: 4, max_input_tokens: 120000}
|
|
|
|
openai_mrcr:
|
|
generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
|
|
run: {limit: 200, concurrency: 4, max_input_tokens: 120000}
|
|
|
|
# ── Agent / tools ──
|
|
bfcl_v3:
|
|
generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0, parallel_tool_calls: true}
|
|
run: {limit_per_task: 20, concurrency: 8, env: bfcl_mock}
|
|
|
|
general_fc:
|
|
generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0}
|
|
run: {limit: 400, concurrency: 8}
|
|
|
|
tau2_bench:
|
|
generation: {temperature: 0.0, max_tokens: 16384, top_p: 1.0}
|
|
run: {concurrency: 8, env: tau2_official, max_turns: 50}
|
|
|
|
swe_bench_verified:
|
|
run: {limit: 70, concurrency: 4}
|