From 0f7c460f1b0a18e848d67490fe1988b47d5d7caa Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 31 Aug 2026 07:24:51 +0000 Subject: [PATCH] Add SciCode sandbox preload and extra bench configs for dpv4. Keep existing yaml entries unchanged; enable SciCode sandbox in run.py and host-side image build without GDPval. Co-authored-by: Cursor --- bash/images_load/preload_k3_sandbox_images.sh | 39 +++++++++++ bash/run.py | 30 +++++++-- config/dpv4-int8_nothinking.yaml | 59 ++++++++++++++++- config/dpv4-int8_thinking.yaml | 65 +++++++++++++++++++ 4 files changed, 188 insertions(+), 5 deletions(-) create mode 100755 bash/images_load/preload_k3_sandbox_images.sh diff --git a/bash/images_load/preload_k3_sandbox_images.sh b/bash/images_load/preload_k3_sandbox_images.sh new file mode 100755 index 0000000..fe8c10f --- /dev/null +++ b/bash/images_load/preload_k3_sandbox_images.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# 在宿主机本地构建 SciCode 沙箱镜像（不从 Docker Hub 拉 scicode-benchmark）。 +# 用法与 Terminal Bench 预加载相同：先在宿主机准备好镜像，再 docker run 评测容器并挂 docker.sock。 +# +# bash bash/images_load/preload_k3_sandbox_images.sh +# +# 注意： +# - 评测仍在 evalscope 容器里跑，沙箱是宿主机 Docker 再起的子容器 +# - SciCode 首次运行还会按内容哈希打 tag，有本地层缓存就不会再拉 Hub +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCICODE_DIR="$PROJECT_ROOT/evalscope/evalscope/benchmarks/scicode/docker" +H5_URL="https://modelscope.cn/datasets/evalscope/SciCode/resolve/master/test_data.h5" + +echo "==> 检查基座镜像（已有则跳过 pull）" +if docker image inspect python:3.11-slim >/dev/null 2>&1; then + echo " ok python:3.11-slim" +else + echo " docker pull python:3.11-slim" + docker pull python:3.11-slim +fi + +echo "==> SciCode: 确保 test_data.h5" +if [ ! -f "$SCICODE_DIR/test_data.h5" ]; then + echo " downloading test_data.h5 ..." + curl -L --fail -o "$SCICODE_DIR/test_data.h5" "$H5_URL" +fi +ls -lh "$SCICODE_DIR/test_data.h5" + +echo "==> docker build scicode-benchmark:latest" +docker build -t scicode-benchmark:latest "$SCICODE_DIR" + +echo "" +echo "完成。本地镜像：" +docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}' | grep -E 'REPOSITORY|scicode-benchmark|python' +echo "" +echo "下一步：docker run 评测镜像，挂 -v /var/run/docker.sock:/var/run/docker.sock" +echo "SciCode 需要 run.py 打开 sandbox（已加入 SANDBOX_DATASETS）。" diff --git a/bash/run.py b/bash/run.py index 024f7ad..c4ea459 100644 --- a/bash/run.py +++ b/bash/run.py @@ -70,6 +70,7 @@ import common as fingerprint_common DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8' DEFAULT_API_URL = 'http://localhost:30000/v1' +DEFAULT_API_KEY = os.environ.get('EVALSCOPE_API_KEY', 'EMPTY') # 数据集缓存根：evalscope 会在其下找 datasets/<名字>-。 # 镜像内通过 EVALSTONE_DATASET_DIR 指到挂载卷，宿主机目录直接命中已有缓存。 DEFAULT_DATASET_DIR = os.environ.get('EVALSTONE_DATASET_DIR', str(PROJECT_ROOT)) @@ -222,7 +223,7 @@ MATH_PROMPT_TEMPLATE = ( "Please reason step by step, and put your final answer within \\boxed{{}}." ) -SANDBOX_DATASETS = {'humaneval', 'bigcodebench'} +SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'scicode'} SANDBOX_CONFIGS = { 'bigcodebench': { 'image': 'bigcodebench-sandbox:latest', @@ -239,6 +240,14 @@ SANDBOX_CONFIGS = { 'python_executor': {} } }, + 'scicode': { + 'image': 'scicode-benchmark:latest', + 'working_dir': '/workspace', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, } @@ -258,12 +267,16 @@ def build_parser(): help='Served model name (default: %(default)s)') parser.add_argument('--api-url', default=DEFAULT_API_URL, help='OpenAI-compatible API URL (default: %(default)s)') + parser.add_argument('--api-key', default=DEFAULT_API_KEY, + help='API key for --api-url (default: env EVALSCOPE_API_KEY or EMPTY)') # Paths parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR, help='Parent directory containing datasets/ subdir (default: %(default)s)') parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR, help='Output root directory (default: %(default)s)') + parser.add_argument('--results-dir', default=None, + help='Directory for summary CSV/Excel (default: /results)') parser.add_argument('--folder-name', default=None, help='Top-level output folder name; defaults to --model, or --model_THINKING when --thinking is enabled') parser.add_argument('--config', default=DEFAULT_CONFIG, @@ -532,6 +545,7 @@ def build_task_config( output_dir: str, model: str, api_url: str, + api_key: str, dataset_dir: str, judge_model_args: dict, run_idx: int = 0, @@ -576,6 +590,7 @@ def build_task_config( return TaskConfig( model=model, api_url=api_url, + api_key=api_key, eval_type='openai_api', dataset_dir=dataset_dir, judge_model_args=judge_model_args, @@ -608,11 +623,15 @@ def build_task_config( # Main # ============================================================ +# Overridden in main() from --results-dir +RESULTS_DIR = PROJECT_ROOT / 'results' + + def write_summary(output_dir: str, model_name: str, folder_name: str, benchmark_names: list = None): """Re-aggregate results for the given benchmarks (or all on disk if None).""" try: - excel_output_dir = PROJECT_ROOT / 'results' + excel_output_dir = Path(RESULTS_DIR) excel_output_dir.mkdir(parents=True, exist_ok=True) if benchmark_names is not None: collect_results_module.eval_benchmark( @@ -705,8 +724,10 @@ def restore_before_run(output_dir: str, benchmark: str, model_name: str, def main(): + global RESULTS_DIR parser = build_parser() args = parser.parse_args() + RESULTS_DIR = Path(args.results_dir) if args.results_dir else (PROJECT_ROOT / 'results') # Resolve limit limit = args.limit @@ -779,7 +800,7 @@ def main(): folder_name = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_') model_output_dir = Path(args.output_dir) / folder_name model_output_dir.mkdir(parents=True, exist_ok=True) - results_dir = PROJECT_ROOT / 'results' + results_dir = RESULTS_DIR results_dir.mkdir(parents=True, exist_ok=True) print('=' * 60) @@ -788,6 +809,7 @@ def main(): print(f'API URL: {args.api_url}') print(f'Dataset Dir: {args.dataset_dir}') print(f'Output Root: {args.output_dir}') + print(f'Summary CSV/Excel: {results_dir}') print(f'Output Folder: {folder_name}') print(f'Suite: {args.suite}') print(f'Limit: {limit if limit is not None else "ALL"}') @@ -905,7 +927,7 @@ def main(): ds_cfg = get_dataset_config(dataset_name) task_cfg = build_task_config( dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args, + str(model_output_dir), args.model, args.api_url, args.api_key, args.dataset_dir, judge_model_args, run_idx=run_idx, thinking_max_tokens_scale=args.thinking_max_tokens_scale, max_tokens_add=args.max_tokens_add, diff --git a/config/dpv4-int8_nothinking.yaml b/config/dpv4-int8_nothinking.yaml index 3e4ee8d..5732d91 100644 --- a/config/dpv4-int8_nothinking.yaml +++ b/config/dpv4-int8_nothinking.yaml @@ -212,4 +212,61 @@ terminal_bench_v2_1: dataset_args: extra_params: timeout_multiplier: 2.0 - max_turns: 500 \ No newline at end of file + max_turns: 500 +aa_lcr: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +scicode: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +deep_swe: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + dataset_args: + extra_params: + pier_model_prefix: openai + pier_agent_kwargs: + model_class: litellm +researchrubrics: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +gdpval: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +tau3_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 16384 + dataset_args: + subset_list: + - banking_knowledge + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 +toolathlon: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 \ No newline at end of file diff --git a/config/dpv4-int8_thinking.yaml b/config/dpv4-int8_thinking.yaml index 05b5221..2841c21 100644 --- a/config/dpv4-int8_thinking.yaml +++ b/config/dpv4-int8_thinking.yaml @@ -234,3 +234,68 @@ swe_bench_pro: stream: true max_tokens: 32768 max_completion_tokens: 64000 +aa_lcr: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +scicode: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +deep_swe: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 + dataset_args: + extra_params: + pier_model_prefix: openai + pier_agent_kwargs: + model_class: litellm +researchrubrics: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +gdpval: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000 +tau3_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 16384 + max_completion_tokens: 64000 + dataset_args: + subset_list: + - banking_knowledge + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 + max_completion_tokens: 64000 +toolathlon: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + max_completion_tokens: 64000