From 1308ddd251c8f6ae794a9ee1e06ed7991605ad05 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Thu, 23 Jul 2026 02:57:53 +0000
Subject: [PATCH] feat(run.py): add --thinking-max-tokens-scale and official
 suite

- Add --thinking-max-tokens-scale to multiply max_tokens when
  --thinking is enabled.
- Add 'official' suite covering all benchmarks in the public
  comparison tables (Kimi/GLM/DS etc).
- Print scale factor in run summary when thinking is on.
---
 bash/run.py | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/bash/run.py b/bash/run.py
index 0cb2dce..a7c2815 100644
--- a/bash/run.py
+++ b/bash/run.py
@@ -148,6 +148,24 @@ SUITES = {
         'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'],
         'agent': ['tau2_bench', 'general_fc'],
     },
+    # 腾讯/官方入库标准对比套件：覆盖各厂商公开对比表里的全部 benchmark
+    # 注：swe_bench_*/terminal_bench_v2/browsecomp/mcp_atlas 需要额外镜像/工具
+    'official': {
+        'multi': [
+            'aime24', 'aime25', 'aime26', 'hmmt26',
+            'imo_answerbench', 'live_code_bench', 'humaneval', 'gpqa_diamond',
+        ],
+        'single': [
+            'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa',
+            'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro',
+            'simple_qa', 'trivia_qa', 'winogrande', 'openai_mrcr', 'longbench_v2',
+            'swe_bench_verified', 'swe_bench_pro', 'swe_bench_multilingual_agentic',
+            'terminal_bench_v2',
+        ],
+        'agent': [
+            'tau2_bench', 'tau2_bench_retail', 'browsecomp', 'mcp_atlas', 'general_fc',
+        ],
+    },
 }
 
 # ============================================================
@@ -230,6 +248,8 @@ def build_parser():
                         help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)')
     parser.add_argument('--no-thinking', dest='thinking', action='store_false',
                         help='Disable thinking mode (default)')
+    parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0,
+                        help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)')
 
     # Judge model
     parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
@@ -395,6 +415,7 @@ def build_task_config(
     dataset_dir: str,
     judge_model_args: dict,
     run_idx: int = 0,
+    thinking_max_tokens_scale: float = 1.0,
 ) -> TaskConfig:
     if run_idx > 0:
         work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
@@ -404,6 +425,11 @@ def build_task_config(
     work_dir = str(work_dir)
 
     generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
+    if enable_thinking and thinking_max_tokens_scale != 1.0:
+        original_max_tokens = generation_config.get('max_tokens', 32768)
+        scaled = int(original_max_tokens * thinking_max_tokens_scale)
+        generation_config['max_tokens'] = scaled
+        print(f'   [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}')
     dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
     dataset_args.setdefault('shuffle', True)
 
@@ -599,6 +625,8 @@ def main():
     print(f'Suite: {args.suite}')
     print(f'Limit: {limit if limit is not None else "ALL"}')
     print(f'Thinking: {enable_thinking}')
+    if enable_thinking:
+        print(f'Thinking max_tokens scale: {args.thinking_max_tokens_scale}')
     print(f'Seed: {args.seed}')
     print(f'Batch Size: {args.batch_size}')
     print(f'Tokenizer Path: {args.tokenizer_path}')
@@ -625,6 +653,7 @@ def main():
             dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
             args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
             run_idx=run_idx,
+            thinking_max_tokens_scale=args.thinking_max_tokens_scale,
         )
         try:
             run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model,