diff --git a/evalharness/cli.py b/evalharness/cli.py index a45d06d..4ba298a 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -800,6 +800,9 @@ def _cmd_eval_run(args) -> int: _shared_reporter.owned_externally = True progress_reporter = _shared_reporter + if getattr(args, 'reasoning_effort', ''): + args._gen_override = {**(getattr(args, '_gen_override', {}) or {}), + 'reasoning_effort': args.reasoning_effort} if model_spec: # generate + score in one go from evalharness.model import run_eval @@ -1207,6 +1210,11 @@ def build_parser() -> argparse.ArgumentParser: "profile.default < profile[''] < explicit kwargs") p.add_argument('--disable-thinking', action='store_true', help='send enable_thinking=false to the OpenAI-compatible model') + p.add_argument('--reasoning-effort', default='', + choices=('', 'minimal', 'low', 'medium', 'high', 'max'), + help="thinking intensity (GLM/Anthropic-style; verified " + "working on the sglang endpoint: low shrinks reasoning " + "~85%%). Overrides the YAML config") p.add_argument('--perf', action='store_true', help='collect streaming TTFT and ITL metrics') p.add_argument('--textools', action='store_true', diff --git a/evalharness/config/effort_low.yaml b/evalharness/config/effort_low.yaml new file mode 100644 index 0000000..9f274b8 --- /dev/null +++ b/evalharness/config/effort_low.yaml @@ -0,0 +1,84 @@ +# 低思考档:全库参数同 default.yaml,仅追加 reasoning_effort: low +# 用法: evalharness eval run ... --config effort_low +default: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + reasoning_effort: low +aime24: + temperature: 1.0 + repeats: 12 + max_tokens: 8192 + reasoning_effort: low +aime25: + temperature: 1.0 + repeats: 12 + max_tokens: 8192 + reasoning_effort: low +aime26: + temperature: 1.0 + repeats: 12 + max_tokens: 8192 + reasoning_effort: low +hmmt26: + temperature: 1.0 + repeats: 12 + max_tokens: 8192 + reasoning_effort: low +imo_answerbench: + temperature: 1.0 + reasoning_effort: low +gpqa_diamond: + temperature: 1.0 + repeats: 3 + max_tokens: 8192 + reasoning_effort: low +mmlu: + max_tokens: 8192 + reasoning_effort: low +mmlu_pro: + max_tokens: 8192 + reasoning_effort: low +cmmlu: + max_tokens: 8192 + reasoning_effort: low +arc: + max_tokens: 8192 + reasoning_effort: low +hellaswag: + max_tokens: 8192 + reasoning_effort: low +winogrande: + max_tokens: 8192 + reasoning_effort: low +simple_qa: + max_tokens: 8192 + reasoning_effort: low +trivia_qa: + max_tokens: 8192 + reasoning_effort: low +humaneval: + temperature: 1.0 + repeats: 3 + reasoning_effort: low +live_code_bench: + temperature: 1.0 + reasoning_effort: low +longbench_v2: + max_tokens: 8192 + max_input_tokens: 128000 + reasoning_effort: low +openai_mrcr: + max_tokens: 8192 + max_input_tokens: 128000 + reasoning_effort: low +bfcl_v3: + max_tokens: 4096 + reasoning_effort: low +general_fc: + max_tokens: 4096 + reasoning_effort: low +tau2_bench: + max_tokens: 16384 + reasoning_effort: low diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 000efc9..078780e 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -380,7 +380,8 @@ class OpenAICompatible(ModelAdapter): ] payload.pop('chat_template_kwargs', None) for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', - 'response_format', 'chat_template_kwargs'): + 'response_format', 'chat_template_kwargs', + 'thinking', 'reasoning_effort'): if kw.get(k) is not None: payload[k] = kw[k] payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room