Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Config: exact copy of evalscope dpv4-int8_nothinking.yaml (generation_config + dataset_args + agent_config, zero custom fields)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 08:35:05 +00:00
parent 3a4be917e6
commit 0ada7d8a0e
2 changed files with 165 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
evalharness/config/README.md
View File

@ -26,7 +26,6 @@ generation:
run: run:
repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用） repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用）
limit: null # 全量 / limit_per_task: 10 每子集 10 条 limit: null # 全量 / limit_per_task: 10 每子集 10 条
concurrency: 4
checkpoint: true checkpoint: true
resume: true resume: true

225
evalharness/config/dp4-nothink.yaml
View File

@ -1,88 +1,193 @@
default:
temperature: 0.0
max_tokens: 32768
top_p: 1.0
gpqa_diamond: gpqa_diamond:
temperature: 1.0 generation_config:
max_tokens: 8192 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 8192
hle: hle:
limit_per_task: 25 generation_config:
judge: dp4-flash temperature: 0.0
judge_url: http://174.1.51.4:30000/v1 top_p: 1.0
stream: true
max_tokens: 32768
aime24: aime24:
temperature: 1.0 generation_config:
repeats: 12 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
aime25: aime25:
temperature: 1.0 generation_config:
repeats: 12 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
mmlu_pro: mmlu_pro:
max_tokens: 8192 generation_config:
limit_per_task: 20 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
simple_qa: simple_qa:
max_tokens: 8192 generation_config:
limit: 200 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
arc: arc:
max_tokens: 8192 generation_config:
limit: 200 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
bbh: bbh:
limit_per_task: 10 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
live_code_bench: live_code_bench:
temperature: 1.0 generation_config:
limit: 200 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
dataset_args:
subset_list:
- release_v6
aime26: aime26:
temperature: 1.0 generation_config:
repeats: 12 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
hmmt26: hmmt26:
temperature: 1.0 generation_config:
repeats: 12 temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
imo_answerbench: imo_answerbench:
temperature: 1.0 generation_config:
limit_per_task: 25 temperature: 1.0
judge: dp4-flash top_p: 1.0
judge_url: http://174.1.51.4:30000/v1 stream: true
max_tokens: 32768
drop: drop:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
hellaswag: hellaswag:
max_tokens: 8192 generation_config:
limit: 400 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
mmlu: mmlu:
max_tokens: 8192 generation_config:
limit_per_task: 4 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
openai_mrcr: openai_mrcr:
max_tokens: 8192 generation_config:
limit: 200 temperature: 0.0
max_input_tokens: 120000 top_p: 1.0
stream: true
max_tokens: 8192
bigcodebench: bigcodebench:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
humaneval: humaneval:
temperature: 1.0 generation_config:
temperature: 1.0
top_p: 1.0
stream: true
max_tokens: 32768
gsm8k: gsm8k:
limit: 200 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
competition_math: competition_math:
limit_per_task: 40 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768
cmmlu: cmmlu:
max_tokens: 8192 generation_config:
limit_per_task: 3 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
trivia_qa: trivia_qa:
max_tokens: 8192 generation_config:
limit: 200 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
winogrande: winogrande:
max_tokens: 8192 generation_config:
limit: 200 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 8192
longbench_v2: longbench_v2:
max_tokens: 8192 generation_config:
limit_per_task: 66 temperature: 0.0
max_input_tokens: 120000 top_p: 1.0
stream: true
max_tokens: 8192
dataset_args:
subset_list:
- short
- medium
- long
tau2_bench: tau2_bench:
max_tokens: 16384 generation_config:
env: tau2_official temperature: 0.0
max_turns: 50 top_p: 1.0
stream: true
max_tokens: 16384
dataset_args:
extra_params:
user_model: deepseek-v4-pro
api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
api_base: https://api.deepseek.com/v1
generation_config:
temperature: 0.0
max_tokens: 4096
agent_config:
mode: native
strategy: react
max_steps: 50
general_fc: general_fc:
max_tokens: 4096 generation_config:
limit: 400 temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 4096
bfcl_v3: bfcl_v3:
max_tokens: 4096 generation_config:
limit_per_task: 20 temperature: 0.0
env: bfcl_mock top_p: 1.0
stream: true
max_tokens: 4096
parallel_tool_calls: true
dataset_args:
extra_params:
is_fc_model: true
underscore_to_dot: true
swe_bench_verified: swe_bench_verified:
limit: 70 generation_config:
temperature: 0.0
top_p: 1.0
stream: true
max_tokens: 32768