From a3251f056f2912dcda04ff171c2e9828bf383fdf Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Thu, 23 Jul 2026 02:40:32 +0000
Subject: [PATCH] feat(run.py): default generation config for unknown
 benchmarks

If a benchmark is not present in the YAML config, use default
parameters instead of skipping:
- temperature: 0.0
- top_p: 1.0
- stream: true
- max_tokens: 32768
---
 bash/run.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/bash/run.py b/bash/run.py
index 2a51d09..0cb2dce 100644
--- a/bash/run.py
+++ b/bash/run.py
@@ -64,6 +64,14 @@ DEFAULT_SEED = 42
 DEFAULT_BATCH_SIZE = 4
 DEFAULT_ENABLE_THINKING = False
 
+# 新 benchmark 没在 YAML 里配时的默认生成参数
+DEFAULT_GENERATION_CONFIG = {
+    'temperature': 0.0,
+    'top_p': 1.0,
+    'stream': True,
+    'max_tokens': 32768,
+}
+
 DEFAULT_JUDGE_MODEL = 'DeepSeek/DeepSeek-V4-Pro'
 DEFAULT_JUDGE_API_URL = 'https://api.vectron.meta-stone.com/v1'
 DEFAULT_JUDGE_API_KEY = 'sk-dbd8a665f7634081b87ec409c7636500'
@@ -601,8 +609,18 @@ def main():
     print(f'Write summary: {args.write_summary}')
     print('=' * 60)
 
+    def get_dataset_config(dataset_name: str) -> dict:
+        """Return configured dataset config, or a default config for unknown benchmarks."""
+        if dataset_name in dataset_configs:
+            return dataset_configs[dataset_name]
+        print(f'WARNING: {dataset_name} not in YAML config, using default generation_config '
+              f'(temperature={DEFAULT_GENERATION_CONFIG["temperature"]}, '
+              f'top_p={DEFAULT_GENERATION_CONFIG["top_p"]}, '
+              f'max_tokens={DEFAULT_GENERATION_CONFIG["max_tokens"]})')
+        return {'generation_config': deepcopy(DEFAULT_GENERATION_CONFIG)}
+
     def run_one(dataset_name, run_idx=0, benchmark_names=None):
-        ds_cfg = dataset_configs[dataset_name]
+        ds_cfg = get_dataset_config(dataset_name)
         task_cfg = build_task_config(
             dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
             args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
@@ -619,9 +637,6 @@ def main():
     benchmark_names = []
 
     for dataset_name in multi_run:
-        if dataset_name not in dataset_configs:
-            print(f'WARNING: {dataset_name} not in YAML config, skipping')
-            continue
         benchmark_names.append(dataset_name)
         num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
         for run_idx in range(num_runs):
@@ -631,9 +646,6 @@ def main():
             run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names)
 
     for dataset_name in single_run:
-        if dataset_name not in dataset_configs:
-            print(f'WARNING: {dataset_name} not in YAML config, skipping')
-            continue
         benchmark_names.append(dataset_name)
         print(f"\n{'='*60}")
         print(f'Running: {dataset_name} (seed={args.seed})')
@@ -641,9 +653,6 @@ def main():
         run_one(dataset_name, benchmark_names=benchmark_names)
 
     for dataset_name in agent:
-        if dataset_name not in dataset_configs:
-            print(f'WARNING: {dataset_name} not in YAML config, skipping')
-            continue
         benchmark_names.append(dataset_name)
         print(f"\n{'='*60}")
         print(f'Running: {dataset_name} (seed={args.seed})')