Add SciCode sandbox preload and extra bench configs for dpv4.

Keep existing yaml entries unchanged; enable SciCode sandbox in run.py and host-side image build without GDPval.

Co-authored-by: Cursor <cursoragent@cursor.com>

bash/images_load/preload_k3_sandbox_images.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# 在宿主机本地构建 SciCode 沙箱镜像（不从 Docker Hub 拉 scicode-benchmark）。
+# 用法与 Terminal Bench 预加载相同：先在宿主机准备好镜像，再 docker run 评测容器并挂 docker.sock。
+#
+#   bash bash/images_load/preload_k3_sandbox_images.sh
+#
+# 注意：
+#   - 评测仍在 evalscope 容器里跑，沙箱是宿主机 Docker 再起的子容器
+#   - SciCode 首次运行还会按内容哈希打 tag，有本地层缓存就不会再拉 Hub
+set -euo pipefail
+
+PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SCICODE_DIR="$PROJECT_ROOT/evalscope/evalscope/benchmarks/scicode/docker"
+H5_URL="https://modelscope.cn/datasets/evalscope/SciCode/resolve/master/test_data.h5"
+
+echo "==> 检查基座镜像（已有则跳过 pull）"
+if docker image inspect python:3.11-slim >/dev/null 2>&1; then
+    echo "    ok python:3.11-slim"
+else
+    echo "    docker pull python:3.11-slim"
+    docker pull python:3.11-slim
+fi
+
+echo "==> SciCode: 确保 test_data.h5"
+if [ ! -f "$SCICODE_DIR/test_data.h5" ]; then
+    echo "    downloading test_data.h5 ..."
+    curl -L --fail -o "$SCICODE_DIR/test_data.h5" "$H5_URL"
+fi
+ls -lh "$SCICODE_DIR/test_data.h5"
+
+echo "==> docker build scicode-benchmark:latest"
+docker build -t scicode-benchmark:latest "$SCICODE_DIR"
+
+echo ""
+echo "完成。本地镜像："
+docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}' | grep -E 'REPOSITORY|scicode-benchmark|python'
+echo ""
+echo "下一步：docker run 评测镜像，挂 -v /var/run/docker.sock:/var/run/docker.sock"
+echo "SciCode 需要 run.py 打开 sandbox（已加入 SANDBOX_DATASETS）。"

bash/run.py

@@ -70,6 +70,7 @@ import common as fingerprint_common
 
 DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8'
 DEFAULT_API_URL = 'http://localhost:30000/v1'
+DEFAULT_API_KEY = os.environ.get('EVALSCOPE_API_KEY', 'EMPTY')
 # 数据集缓存根：evalscope 会在其下找 datasets/<名字>-<hash>。
 # 镜像内通过 EVALSTONE_DATASET_DIR 指到挂载卷，宿主机目录直接命中已有缓存。
 DEFAULT_DATASET_DIR = os.environ.get('EVALSTONE_DATASET_DIR', str(PROJECT_ROOT))
@@ -222,7 +223,7 @@ MATH_PROMPT_TEMPLATE = (
     "Please reason step by step, and put your final answer within \\boxed{{}}."
 )
 
-SANDBOX_DATASETS = {'humaneval', 'bigcodebench'}
+SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'scicode'}
 SANDBOX_CONFIGS = {
     'bigcodebench': {
         'image': 'bigcodebench-sandbox:latest',
@@ -239,6 +240,14 @@ SANDBOX_CONFIGS = {
             'python_executor': {}
         }
     },
+    'scicode': {
+        'image': 'scicode-benchmark:latest',
+        'working_dir': '/workspace',
+        'tools_config': {
+            'shell_executor': {},
+            'python_executor': {}
+        }
+    },
 }
 
 
@@ -258,12 +267,16 @@ def build_parser():
                         help='Served model name (default: %(default)s)')
     parser.add_argument('--api-url', default=DEFAULT_API_URL,
                         help='OpenAI-compatible API URL (default: %(default)s)')
+    parser.add_argument('--api-key', default=DEFAULT_API_KEY,
+                        help='API key for --api-url (default: env EVALSCOPE_API_KEY or EMPTY)')
 
     # Paths
     parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR,
                         help='Parent directory containing datasets/ subdir (default: %(default)s)')
     parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR,
                         help='Output root directory (default: %(default)s)')
+    parser.add_argument('--results-dir', default=None,
+                        help='Directory for summary CSV/Excel (default: <repo>/results)')
     parser.add_argument('--folder-name', default=None,
                         help='Top-level output folder name; defaults to --model, or --model_THINKING when --thinking is enabled')
     parser.add_argument('--config', default=DEFAULT_CONFIG,
@@ -532,6 +545,7 @@ def build_task_config(
     output_dir: str,
     model: str,
     api_url: str,
+    api_key: str,
     dataset_dir: str,
     judge_model_args: dict,
     run_idx: int = 0,
@@ -576,6 +590,7 @@ def build_task_config(
     return TaskConfig(
         model=model,
         api_url=api_url,
+        api_key=api_key,
         eval_type='openai_api',
         dataset_dir=dataset_dir,
         judge_model_args=judge_model_args,
@@ -608,11 +623,15 @@ def build_task_config(
 # Main
 # ============================================================
 
+# Overridden in main() from --results-dir
+RESULTS_DIR = PROJECT_ROOT / 'results'
+
+
 def write_summary(output_dir: str, model_name: str, folder_name: str,
                   benchmark_names: list = None):
     """Re-aggregate results for the given benchmarks (or all on disk if None)."""
     try:
-        excel_output_dir = PROJECT_ROOT / 'results'
+        excel_output_dir = Path(RESULTS_DIR)
         excel_output_dir.mkdir(parents=True, exist_ok=True)
         if benchmark_names is not None:
             collect_results_module.eval_benchmark(
@@ -705,8 +724,10 @@ def restore_before_run(output_dir: str, benchmark: str, model_name: str,
 
 
 def main():
+    global RESULTS_DIR
     parser = build_parser()
     args = parser.parse_args()
+    RESULTS_DIR = Path(args.results_dir) if args.results_dir else (PROJECT_ROOT / 'results')
 
     # Resolve limit
     limit = args.limit
@@ -779,7 +800,7 @@ def main():
         folder_name = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_')
     model_output_dir = Path(args.output_dir) / folder_name
     model_output_dir.mkdir(parents=True, exist_ok=True)
-    results_dir = PROJECT_ROOT / 'results'
+    results_dir = RESULTS_DIR
     results_dir.mkdir(parents=True, exist_ok=True)
 
     print('=' * 60)
@@ -788,6 +809,7 @@ def main():
     print(f'API URL: {args.api_url}')
     print(f'Dataset Dir: {args.dataset_dir}')
     print(f'Output Root: {args.output_dir}')
+    print(f'Summary CSV/Excel: {results_dir}')
     print(f'Output Folder: {folder_name}')
     print(f'Suite: {args.suite}')
     print(f'Limit: {limit if limit is not None else "ALL"}')
@@ -905,7 +927,7 @@ def main():
         ds_cfg = get_dataset_config(dataset_name)
         task_cfg = build_task_config(
             dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
-            str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args,
+            str(model_output_dir), args.model, args.api_url, args.api_key, args.dataset_dir, judge_model_args,
             run_idx=run_idx,
             thinking_max_tokens_scale=args.thinking_max_tokens_scale,
             max_tokens_add=args.max_tokens_add,

config/dpv4-int8_nothinking.yaml

@@ -213,3 +213,60 @@ terminal_bench_v2_1:
       extra_params:                                                                                                                                                                                                                 
         timeout_multiplier: 2.0                                                                                                                                                                                                     
         max_turns: 500    
+aa_lcr:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+scicode:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+deep_swe:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+  dataset_args:
+    extra_params:
+      pier_model_prefix: openai
+      pier_agent_kwargs:
+        model_class: litellm
+researchrubrics:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+gdpval:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+tau3_bench:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 16384
+  dataset_args:
+    subset_list:
+    - banking_knowledge
+    extra_params:
+      user_model: deepseek-v4-pro
+      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
+      api_base: https://api.deepseek.com/v1
+      generation_config:
+        temperature: 0.0
+        max_tokens: 4096
+toolathlon:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768

config/dpv4-int8_thinking.yaml

@@ -234,3 +234,68 @@ swe_bench_pro:
     stream: true
     max_tokens: 32768
     max_completion_tokens: 64000
+aa_lcr:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000
+scicode:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000
+deep_swe:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000
+  dataset_args:
+    extra_params:
+      pier_model_prefix: openai
+      pier_agent_kwargs:
+        model_class: litellm
+researchrubrics:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000
+gdpval:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000
+tau3_bench:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 16384
+    max_completion_tokens: 64000
+  dataset_args:
+    subset_list:
+    - banking_knowledge
+    extra_params:
+      user_model: deepseek-v4-pro
+      api_key: sk-9ed86ef546ca47e3afa7c3b014dea268
+      api_base: https://api.deepseek.com/v1
+      generation_config:
+        temperature: 0.0
+        max_tokens: 4096
+        max_completion_tokens: 64000
+toolathlon:
+  generation_config:
+    temperature: 1.0
+    top_p: 1.0
+    stream: true
+    max_tokens: 32768
+    max_completion_tokens: 64000