From 0ada7d8a0e1ce15005f390a37552586429fbf3c8 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 11 Sep 2026 08:35:05 +0000 Subject: [PATCH] Config: exact copy of evalscope dpv4-int8_nothinking.yaml (generation_config + dataset_args + agent_config, zero custom fields) Co-Authored-By: Claude --- evalharness/config/README.md | 1 - evalharness/config/dp4-nothink.yaml | 225 ++++++++++++++++++++-------- 2 files changed, 165 insertions(+), 61 deletions(-) diff --git a/evalharness/config/README.md b/evalharness/config/README.md index e3c02a6..3fbc609 100644 --- a/evalharness/config/README.md +++ b/evalharness/config/README.md @@ -26,7 +26,6 @@ generation: run: repeats: 12 # 跑 12 遍取均值（temp=1 方差测量用） limit: null # 全量 / limit_per_task: 10 每子集 10 条 - concurrency: 4 checkpoint: true resume: true diff --git a/evalharness/config/dp4-nothink.yaml b/evalharness/config/dp4-nothink.yaml index 0a031f5..bc84403 100644 --- a/evalharness/config/dp4-nothink.yaml +++ b/evalharness/config/dp4-nothink.yaml @@ -1,88 +1,193 @@ -default: - temperature: 0.0 - max_tokens: 32768 - top_p: 1.0 gpqa_diamond: - temperature: 1.0 - max_tokens: 8192 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 8192 hle: - limit_per_task: 25 - judge: dp4-flash - judge_url: http://174.1.51.4:30000/v1 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 aime24: - temperature: 1.0 - repeats: 12 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 aime25: - temperature: 1.0 - repeats: 12 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 mmlu_pro: - max_tokens: 8192 - limit_per_task: 20 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 simple_qa: - max_tokens: 8192 - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 arc: - max_tokens: 8192 - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 bbh: - limit_per_task: 10 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 live_code_bench: - temperature: 1.0 - limit: 200 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + subset_list: + - release_v6 aime26: - temperature: 1.0 - repeats: 12 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 hmmt26: - temperature: 1.0 - repeats: 12 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 imo_answerbench: - temperature: 1.0 - limit_per_task: 25 - judge: dp4-flash - judge_url: http://174.1.51.4:30000/v1 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 drop: - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 hellaswag: - max_tokens: 8192 - limit: 400 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 mmlu: - max_tokens: 8192 - limit_per_task: 4 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 openai_mrcr: - max_tokens: 8192 - limit: 200 - max_input_tokens: 120000 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 bigcodebench: - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 humaneval: - temperature: 1.0 + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 gsm8k: - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 competition_math: - limit_per_task: 40 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 cmmlu: - max_tokens: 8192 - limit_per_task: 3 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 trivia_qa: - max_tokens: 8192 - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 winogrande: - max_tokens: 8192 - limit: 200 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 longbench_v2: - max_tokens: 8192 - limit_per_task: 66 - max_input_tokens: 120000 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 + dataset_args: + subset_list: + - short + - medium + - long tau2_bench: - max_tokens: 16384 - env: tau2_official - max_turns: 50 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 16384 + dataset_args: + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 + agent_config: + mode: native + strategy: react + max_steps: 50 general_fc: - max_tokens: 4096 - limit: 400 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 bfcl_v3: - max_tokens: 4096 - limit_per_task: 20 - env: bfcl_mock + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 + parallel_tool_calls: true + dataset_args: + extra_params: + is_fc_model: true + underscore_to_dot: true swe_bench_verified: - limit: 70 + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768