sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(run.py): add --max-tokens-add parameter

Browse Source 
Allow adding a fixed number of tokens to every benchmark's max_tokens
on top of the configured value. Applied after --thinking-max-tokens-scale.
Update myread.md with examples.
This commit is contained in:
sora 2026-07-23 03:18:30 +00:00
parent a8c6fbdc75
commit 0c72d89d1c
2 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
bash/run.py
View File

@ -252,6 +252,8 @@ def build_parser():
help='Disable thinking mode (default)') help='Disable thinking mode (default)')
parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0, parser.add_argument('--thinking-max-tokens-scale', type=float, default=1.0,
help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)') help='Scale max_tokens by this factor when --thinking is enabled (default: %(default)s)')
parser.add_argument('--max-tokens-add', type=int, default=0,
help='Add this many tokens to every benchmark max_tokens (applied after scale)')
# Judge model # Judge model
parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL, parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
@ -418,6 +420,7 @@ def build_task_config(
judge_model_args: dict, judge_model_args: dict,
run_idx: int = 0, run_idx: int = 0,
thinking_max_tokens_scale: float = 1.0, thinking_max_tokens_scale: float = 1.0,
max_tokens_add: int = 0,
) -> TaskConfig: ) -> TaskConfig:
if run_idx > 0: if run_idx > 0:
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}' work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
@ -432,6 +435,11 @@ def build_task_config(
scaled = int(original_max_tokens * thinking_max_tokens_scale) scaled = int(original_max_tokens * thinking_max_tokens_scale)
generation_config['max_tokens'] = scaled generation_config['max_tokens'] = scaled
print(f' [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}') print(f' [thinking] max_tokens scaled: {original_max_tokens} -> {scaled}')
if max_tokens_add > 0:
original_max_tokens = generation_config.get('max_tokens', 32768)
new_max_tokens = original_max_tokens + max_tokens_add
generation_config['max_tokens'] = new_max_tokens
print(f' max_tokens add: {original_max_tokens} -> {new_max_tokens}')
dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
dataset_args.setdefault('shuffle', True) dataset_args.setdefault('shuffle', True)
@ -669,6 +677,7 @@ def main():
str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args, str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args,
run_idx=run_idx, run_idx=run_idx,
thinking_max_tokens_scale=args.thinking_max_tokens_scale, thinking_max_tokens_scale=args.thinking_max_tokens_scale,
max_tokens_add=args.max_tokens_add,
) )
try: try:
run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model, run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model,

4
myread.md
View File

@ -376,6 +376,7 @@ python bash/run.py --model MyModel --api-url http://10.0.0.5:30000/v1
| `--thinking` | `False` | 开启 sglang thinking 模式 | | `--thinking` | `False` | 开启 sglang thinking 模式 |
| `--no-thinking` | `False` | 关闭 thinking 模式（默认） | | `--no-thinking` | `False` | 关闭 thinking 模式（默认） |
| `--thinking-max-tokens-scale` | `1.0` | `--thinking` 开启时，max_tokens 乘以该系数 | | `--thinking-max-tokens-scale` | `1.0` | `--thinking` 开启时，max_tokens 乘以该系数 |
| `--max-tokens-add` | `0` | 在每个 benchmark 原 max_tokens 基础上直接增加 N tokens |
| `--judge-model` | `DeepSeek/DeepSeek-V4-Pro` | 裁判模型名（LLM-as-judge） | | `--judge-model` | `DeepSeek/DeepSeek-V4-Pro` | 裁判模型名（LLM-as-judge） |
| `--judge-api-url` | Vectron 地址 | 裁判模型 API 地址 | | `--judge-api-url` | Vectron 地址 | 裁判模型 API 地址 |
| `--judge-api-key` | 内置 key | 裁判模型 API key | | `--judge-api-key` | 内置 key | 裁判模型 API key |
@ -391,6 +392,9 @@ python bash/run.py --model MyModel --api-url http://10.0.0.5:30000/v1
# 开启 thinking，max_tokens 自动翻倍 # 开启 thinking，max_tokens 自动翻倍
python bash/run.py --suite full --thinking --thinking-max-tokens-scale 2.0 --limit none python bash/run.py --suite full --thinking --thinking-max-tokens-scale 2.0 --limit none
# 开启 thinking，每个 benchmark 原 max_tokens 再加 128k
python bash/run.py --suite full --thinking --max-tokens-add 128000 --limit none
# 自定义输出文件夹名 # 自定义输出文件夹名
python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none python bash/run.py --suite full --thinking --folder-name my_exp_v1 --limit none
``` ```