feat(run.py): add --thinking-max-tokens-scale and official suite
- Add --thinking-max-tokens-scale to multiply max_tokens when --thinking is enabled. - Add 'official' suite covering all benchmarks in the public comparison tables (Kimi/GLM/DS etc). - Print scale factor in run summary when thinking is on.
This commit is contained in:
parent
a3251f056f
commit
1308ddd251
29
bash/run.py
29
bash/run.py
@ -148,6 +148,24 @@ SUITES = {
|
||||
'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'],
|
||||
'agent': ['tau2_bench', 'general_fc'],
|
||||
},
|
||||
# 腾讯/官方入库标准对比套件:覆盖各厂商公开对比表里的全部 benchmark
|
||||
# 注:swe_bench_*/terminal_bench_v2/browsecomp/mcp_atlas 需要额外镜像/工具
|
||||
'official': {
|
||||
'multi': [
|
||||
'aime24', 'aime25', 'aime26', 'hmmt26',
|
||||
'imo_answerbench', 'live_code_bench', 'humaneval', 'gpqa_diamond',
|
||||
],
|
||||
'single': [
|
||||
'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa',
|
||||
'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro',
|
||||
'simple_qa', 'trivia_qa', 'winogrande', 'openai_mrcr', 'longbench_v2',
|
||||
'swe_bench_verified', 'swe_bench_pro', 'swe_bench_multilingual_agentic',
|
||||
'terminal_bench_v2',
|
||||
],
|
||||
'agent': [
|
||||
'tau2_bench', 'tau2_bench_retail', 'browsecomp', 'mcp_atlas', 'general_fc',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
@ -230,6 +248,8 @@ def build_parser():
|
||||
help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)')
|
||||
parser.add_argument('--no-thinking', dest='thinking', action='store_false',
|
||||
help='Disable thinking mode (default)')
|
||||
parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0,
|
||||
help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)')
|
||||
|
||||
# Judge model
|
||||
parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
|
||||
@ -395,6 +415,7 @@ def build_task_config(
|
||||
dataset_dir: str,
|
||||
judge_model_args: dict,
|
||||
run_idx: int = 0,
|
||||
thinking_max_tokens_scale: float = 1.0,
|
||||
) -> TaskConfig:
|
||||
if run_idx > 0:
|
||||
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
|
||||
@ -404,6 +425,11 @@ def build_task_config(
|
||||
work_dir = str(work_dir)
|
||||
|
||||
generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
|
||||
if enable_thinking and thinking_max_tokens_scale != 1.0:
|
||||
original_max_tokens = generation_config.get('max_tokens', 32768)
|
||||
scaled = int(original_max_tokens * thinking_max_tokens_scale)
|
||||
generation_config['max_tokens'] = scaled
|
||||
print(f' [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}')
|
||||
dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
|
||||
dataset_args.setdefault('shuffle', True)
|
||||
|
||||
@ -599,6 +625,8 @@ def main():
|
||||
print(f'Suite: {args.suite}')
|
||||
print(f'Limit: {limit if limit is not None else "ALL"}')
|
||||
print(f'Thinking: {enable_thinking}')
|
||||
if enable_thinking:
|
||||
print(f'Thinking max_tokens scale: {args.thinking_max_tokens_scale}')
|
||||
print(f'Seed: {args.seed}')
|
||||
print(f'Batch Size: {args.batch_size}')
|
||||
print(f'Tokenizer Path: {args.tokenizer_path}')
|
||||
@ -625,6 +653,7 @@ def main():
|
||||
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
|
||||
args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
|
||||
run_idx=run_idx,
|
||||
thinking_max_tokens_scale=args.thinking_max_tokens_scale,
|
||||
)
|
||||
try:
|
||||
run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user