feat(run.py): add --thinking-max-tokens-scale and official suite

- Add --thinking-max-tokens-scale to multiply max_tokens when
  --thinking is enabled.
- Add 'official' suite covering all benchmarks in the public
  comparison tables (Kimi/GLM/DS etc).
- Print scale factor in run summary when thinking is on.
This commit is contained in:
sora 2026-07-23 02:57:53 +00:00
parent a3251f056f
commit 1308ddd251

View File

@ -148,6 +148,24 @@ SUITES = {
'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'],
'agent': ['tau2_bench', 'general_fc'],
},
# 腾讯/官方入库标准对比套件:覆盖各厂商公开对比表里的全部 benchmark
# 注swe_bench_*/terminal_bench_v2/browsecomp/mcp_atlas 需要额外镜像/工具
'official': {
'multi': [
'aime24', 'aime25', 'aime26', 'hmmt26',
'imo_answerbench', 'live_code_bench', 'humaneval', 'gpqa_diamond',
],
'single': [
'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa',
'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro',
'simple_qa', 'trivia_qa', 'winogrande', 'openai_mrcr', 'longbench_v2',
'swe_bench_verified', 'swe_bench_pro', 'swe_bench_multilingual_agentic',
'terminal_bench_v2',
],
'agent': [
'tau2_bench', 'tau2_bench_retail', 'browsecomp', 'mcp_atlas', 'general_fc',
],
},
}
# ============================================================
@ -230,6 +248,8 @@ def build_parser():
help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)')
parser.add_argument('--no-thinking', dest='thinking', action='store_false',
help='Disable thinking mode (default)')
parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0,
help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)')
# Judge model
parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
@ -395,6 +415,7 @@ def build_task_config(
dataset_dir: str,
judge_model_args: dict,
run_idx: int = 0,
thinking_max_tokens_scale: float = 1.0,
) -> TaskConfig:
if run_idx > 0:
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
@ -404,6 +425,11 @@ def build_task_config(
work_dir = str(work_dir)
generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
if enable_thinking and thinking_max_tokens_scale != 1.0:
original_max_tokens = generation_config.get('max_tokens', 32768)
scaled = int(original_max_tokens * thinking_max_tokens_scale)
generation_config['max_tokens'] = scaled
print(f' [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}')
dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
dataset_args.setdefault('shuffle', True)
@ -599,6 +625,8 @@ def main():
print(f'Suite: {args.suite}')
print(f'Limit: {limit if limit is not None else "ALL"}')
print(f'Thinking: {enable_thinking}')
if enable_thinking:
print(f'Thinking max_tokens scale: {args.thinking_max_tokens_scale}')
print(f'Seed: {args.seed}')
print(f'Batch Size: {args.batch_size}')
print(f'Tokenizer Path: {args.tokenizer_path}')
@ -625,6 +653,7 @@ def main():
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
run_idx=run_idx,
thinking_max_tokens_scale=args.thinking_max_tokens_scale,
)
try:
run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model,