diff --git a/bash/run.py b/bash/run.py index 303e4fa..0b229ca 100644 --- a/bash/run.py +++ b/bash/run.py @@ -252,6 +252,8 @@ def build_parser(): help='Disable thinking mode (default)') parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0, help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)') + parser.add_argument('--max-tokens-add', type=int, default=0, + help='Add this many tokens to every benchmark max_tokens (applied after scale)') # Judge model parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL, @@ -418,6 +420,7 @@ def build_task_config( judge_model_args: dict, run_idx: int = 0, thinking_max_tokens_scale: float = 1.0, + max_tokens_add: int = 0, ) -> TaskConfig: if run_idx > 0: work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}' @@ -432,6 +435,11 @@ def build_task_config( scaled = int(original_max_tokens * thinking_max_tokens_scale) generation_config['max_tokens'] = scaled print(f' [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}') + if max_tokens_add > 0: + original_max_tokens = generation_config.get('max_tokens', 32768) + new_max_tokens = original_max_tokens + max_tokens_add + generation_config['max_tokens'] = new_max_tokens + print(f' max_tokens add: {original_max_tokens} -> {new_max_tokens}') dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) dataset_args.setdefault('shuffle', True) @@ -669,6 +677,7 @@ def main(): str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args, run_idx=run_idx, thinking_max_tokens_scale=args.thinking_max_tokens_scale, + max_tokens_add=args.max_tokens_add, ) try: run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model, diff --git a/myread.md b/myread.md index e85dd54..951561d 100644 --- a/myread.md +++ b/myread.md @@ -376,6 +376,7 @@ python bash/run.py --model MyModel --api-url http://10.0.0.5:30000/v1 | `--thinking` | `False` | 开启 sglang thinking 模式 | | `--no-thinking` | `False` | 关闭 thinking 模式(默认) | | `--thinking-max-tokens-scale` | `1.0` | `--thinking` 开启时,max_tokens 乘以该系数 | +| `--max-tokens-add` | `0` | 在每个 benchmark 原 max_tokens 基础上直接增加 N tokens | | `--judge-model` | `DeepSeek/DeepSeek-V4-Pro` | 裁判模型名(LLM-as-judge) | | `--judge-api-url` | Vectron 地址 | 裁判模型 API 地址 | | `--judge-api-key` | 内置 key | 裁判模型 API key | @@ -391,6 +392,9 @@ python bash/run.py --model MyModel --api-url http://10.0.0.5:30000/v1 # 开启 thinking,max_tokens 自动翻倍 python bash/run.py --suite full --thinking --thinking-max-tokens-scale 2.0 --limit none +# 开启 thinking,每个 benchmark 原 max_tokens 再加 128k +python bash/run.py --suite full --thinking --max-tokens-add 128000 --limit none + # 自定义输出文件夹名 python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none ```