Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Config: exact copy of evalscope dpv4-int8_nothinking.yaml (generation_config + dataset_args + agent_config, zero custom fields)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 08:35:05 +00:00
parent 3a4be917e6
commit 0ada7d8a0e
2 changed files with 165 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
evalharness/config/README.md
View File

@ -26,7 +26,6 @@ generation:
run: run:
repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用） repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用）
limit: null # 全量 / limit_per_task: 10 每子集 10 条 limit: null # 全量 / limit_per_task: 10 每子集 10 条
concurrency: 4
checkpoint: true checkpoint: true
resume: true resume: true

181
evalharness/config/dp4-nothink.yaml
View File

@ -1,88 +1,193 @@
default:
temperature: 0.0
max_tokens: 32768
top_p: 1.0
gpqa_diamond: gpqa_diamond:
generation_config:
temperature: 1.0 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
hle: hle:
limit_per_task: 25 generation_config:
judge: dp4-flash temperature: 0.0
judge_url: http://174.1.51.4:30000/v1 top_p: 1.0
stream: true
max_tokens: 32768
aime24: aime24:
generation_config:
temperature: 1.0 temperature: 1.0
repeats: 12 top_p: 1.0
stream: true
max_tokens: 32768
aime25: aime25:
generation_config:
temperature: 1.0 temperature: 1.0
repeats: 12 top_p: 1.0
stream: true
max_tokens: 32768
mmlu_pro: mmlu_pro:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit_per_task: 20
simple_qa: simple_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 200
arc: arc:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 200
bbh: bbh:
limit_per_task: 10 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
live_code_bench: live_code_bench:
generation_config:
temperature: 1.0 temperature: 1.0
limit: 200 top_p: 1.0
stream: true
max_tokens: 32768
dataset_args:
subset_list:
- release_v6
aime26: aime26:
generation_config:
temperature: 1.0 temperature: 1.0
repeats: 12 top_p: 1.0
stream: true
max_tokens: 32768
hmmt26: hmmt26:
generation_config:
temperature: 1.0 temperature: 1.0
repeats: 12 top_p: 1.0
stream: true
max_tokens: 32768
imo_answerbench: imo_answerbench:
generation_config:
temperature: 1.0 temperature: 1.0
limit_per_task: 25 top_p: 1.0
judge: dp4-flash stream: true
judge_url: http://174.1.51.4:30000/v1 max_tokens: 32768
drop: drop:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
hellaswag: hellaswag:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 400
mmlu: mmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit_per_task: 4
openai_mrcr: openai_mrcr:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 200
max_input_tokens: 120000
bigcodebench: bigcodebench:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
humaneval: humaneval:
generation_config:
temperature: 1.0 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
gsm8k: gsm8k:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
competition_math: competition_math:
limit_per_task: 40 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
cmmlu: cmmlu:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit_per_task: 3
trivia_qa: trivia_qa:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 200
winogrande: winogrande:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit: 200
longbench_v2: longbench_v2:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192 max_tokens: 8192
limit_per_task: 66 dataset_args:
max_input_tokens: 120000 subset_list:
- short
- medium
- long
tau2_bench: tau2_bench:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 16384 max_tokens: 16384
env: tau2_official dataset_args:
max_turns: 50 extra_params:
user_model: deepseek-v4-pro
api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
api_base: https://api.deepseek.com/v1
generation_config:
temperature: 0.0
max_tokens: 4096
agent_config:
mode: native
strategy: react
max_steps: 50
general_fc: general_fc:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096 max_tokens: 4096
limit: 400
bfcl_v3: bfcl_v3:
generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096 max_tokens: 4096
limit_per_task: 20 parallel_tool_calls: true
env: bfcl_mock dataset_args:
extra_params:
is_fc_model: true
underscore_to_dot: true
swe_bench_verified: swe_bench_verified:
limit: 70 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768