# dp4-nothink: DeepSeek-V4-Flash nothinking protocol
# Usage: evalharness eval run <bench> --config dp4-nothink --api-url ... --model ...
# Priority: DatasetSpec.gen_config < default < <bench> < CLI args

default:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0, stream: true}
  run: {concurrency: 8, checkpoint: true, resume: true}

# ── Math (temp=1, 12 runs averaged) ──
aime24:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {repeats: 12, concurrency: 4}

aime25:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {repeats: 12, concurrency: 4}

aime26:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {repeats: 12, concurrency: 4}

hmmt26:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {repeats: 12, concurrency: 4}

competition_math:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
  run: {limit_per_task: 40, concurrency: 8}

gsm8k:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
  run: {limit: 200, concurrency: 8}

# ── Knowledge / MCQ ──
mmlu:
  run: {limit_per_task: 4, concurrency: 8}

cmmlu:
  run: {limit_per_task: 3, concurrency: 8}

mmlu_pro:
  run: {limit_per_task: 20, concurrency: 8}

arc:
  run: {limit: 200, concurrency: 8}

hellaswag:
  run: {limit: 400, concurrency: 8}

winogrande:
  run: {limit: 200, concurrency: 8}

bbh:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
  run: {limit_per_task: 10, concurrency: 8}

gpqa_diamond:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {limit: 200, concurrency: 8}

# ── QA ──
simple_qa:
  generation: {temperature: 0.0, max_tokens: 1024, top_p: 1.0}
  run: {limit: 200, concurrency: 8}

trivia_qa:
  run: {limit: 200, concurrency: 8}

drop:
  run: {limit: 200, concurrency: 8}

# ── Judged ──
hle:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
  run: {limit_per_task: 25, concurrency: 8}
  judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}

imo_answerbench:
  generation: {temperature: 1.0, max_tokens: 8192, top_p: 1.0}
  run: {limit_per_task: 25, concurrency: 8}
  judge: {model: dp4-flash, api_url: 'http://174.1.51.4:30000/v1'}

# ── Code (docker sandbox) ──
humaneval:
  generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
  run: {concurrency: 4}

bigcodebench:
  generation: {temperature: 0.0, max_tokens: 32768, top_p: 1.0}
  run: {limit: 200, concurrency: 4}

live_code_bench:
  generation: {temperature: 1.0, max_tokens: 32768, top_p: 1.0}
  run: {limit: 200, concurrency: 4}

# ── Long context ──
longbench_v2:
  generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
  run: {limit_per_task: 66, concurrency: 4, max_input_tokens: 120000}

openai_mrcr:
  generation: {temperature: 0.0, max_tokens: 8192, top_p: 1.0}
  run: {limit: 200, concurrency: 4, max_input_tokens: 120000}

# ── Agent / tools ──
bfcl_v3:
  generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0, parallel_tool_calls: true}
  run: {limit_per_task: 20, concurrency: 8, env: bfcl_mock}

general_fc:
  generation: {temperature: 0.0, max_tokens: 4096, top_p: 1.0}
  run: {limit: 400, concurrency: 8}

tau2_bench:
  generation: {temperature: 0.0, max_tokens: 16384, top_p: 1.0}
  run: {concurrency: 8, env: tau2_official, max_turns: 50}

swe_bench_verified:
  run: {limit: 70, concurrency: 4}