From 991a4a5f7f39af53b9d2d79afabdf45d7dcfdd52 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 27 Jul 2026 05:34:16 +0000 Subject: [PATCH] Add sandbox images download from ModelScope; fix Excel sample count total --- README.md | 170 ++++- bash/case/DP4-flash-int8-no-thinking-full.sh | 1 + bash/case/DP4-flash-int8-thinking-official.sh | 6 + bash/collect_results.py | 9 +- bash/run.py | 8 +- bash/test.py | 659 ------------------ bash/test_e2e.py | 179 ----- .../evalscope/benchmarks/swe_bench/utils.py | 22 + myread.md | 110 ++- scripts/Dockerfile.bigcodebench-sandbox | 58 ++ scripts/deploy_evalscope.sh | 292 ++++++++ 11 files changed, 643 insertions(+), 871 deletions(-) create mode 100644 bash/case/DP4-flash-int8-no-thinking-full.sh create mode 100644 bash/case/DP4-flash-int8-thinking-official.sh delete mode 100644 bash/test.py delete mode 100644 bash/test_e2e.py create mode 100644 scripts/Dockerfile.bigcodebench-sandbox create mode 100755 scripts/deploy_evalscope.sh diff --git a/README.md b/README.md index d8b0b0c..81859d8 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,163 @@ python bash/run.py \ --- -## 3. `bash/run.py` 完整参数 +## 3. Docker 镜像下载与部署 + +根据是否跑 SWE-bench，选择下面两种方式之一。 + +### 3.1 方式 A：不测 SWE-bench 的 Docker 配置 + +如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像： + +```bash +# 1. 暂停服务 +sudo systemctl stop docker.socket +sudo systemctl stop docker + +# 2. 配置 Docker 国内镜像加速 +sudo tee /etc/docker/daemon.json <<-'EOF' +{ + "registry-mirrors": [ + "https://docker.m.daocloud.io", + "https://docker.1ms.run", + "https://hub.rat.dev", + "https://docker.1panel.live", + "https://dockerproxy.com", + "https://hub-mirror.c.163.com", + "https://mirror.baidubce.com", + "https://docker.mirrors.ustc.edu.cn", + "https://docker.mirrors.sjtug.sjtu.edu.cn", + "https://docker.nju.edu.cn", + "https://docker.mirrors.tuna.tsinghua.edu.cn" + ] +} +EOF + +# 3. 重启 Docker +sudo systemctl start docker + +# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖） +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz + +# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须） +# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时） +python3 -c " +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='bigcodebench-sandbox.tar.gz', + local_dir='/data1/sora/evalscope/docker/sandbox_images' +) +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='python-3.11-slim.tar.gz', + local_dir='/data1/sora/evalscope/docker/sandbox_images' +) +" +docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz +docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz + +# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取 +# docker pull bigcodebench/bigcodebench-evaluate:latest +# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF' +# FROM bigcodebench/bigcodebench-evaluate:latest +# ENTRYPOINT [] +# CMD ["tail", "-f", "/dev/null"] +# EOF +# docker pull python:3.11-slim +``` + +验证： + +```bash +docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim' +``` + +### 3.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径） + +SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘： + +```bash +# 1. 暂停服务 +sudo systemctl stop docker.socket +sudo systemctl stop docker +sudo systemctl stop containerd + +# 2. 创建新的数据目录 +mkdir -p /data1/sora/evalscope/docker/images +mkdir -p /data1/sora/evalscope/docker/containerd + +# 3. 配置国内镜像加速 + 大容量 data-root +sudo tee /etc/docker/daemon.json <<-'EOF' +{ + "data-root": "/data1/sora/evalscope/docker/images", + "registry-mirrors": [ + "https://docker.m.daocloud.io", + "https://docker.1ms.run", + "https://hub.rat.dev", + "https://docker.1panel.live", + "https://dockerproxy.com", + "https://hub-mirror.c.163.com", + "https://mirror.baidubce.com", + "https://docker.mirrors.ustc.edu.cn", + "https://docker.mirrors.sjtug.sjtu.edu.cn", + "https://docker.nju.edu.cn", + "https://docker.mirrors.tuna.tsinghua.edu.cn" + ] +} +EOF + +# 4. 配置 containerd 数据目录 +sudo tee /etc/containerd/config.toml <<-'EOF' +root = "/data1/sora/evalscope/docker/containerd" +state = "/run/containerd" +EOF + +# 5. 重启 +sudo systemctl reset-failed docker.service +sudo systemctl start containerd +sudo systemctl start docker + +# 6. 加载 evalscope 环境镜像 +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz + +# 7. 准备代码执行 sandbox 镜像 +# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时） +python3 -c " +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='bigcodebench-sandbox.tar.gz', + local_dir='/data1/sora/evalscope/docker/sandbox_images' +) +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='python-3.11-slim.tar.gz', + local_dir='/data1/sora/evalscope/docker/sandbox_images' +) +" +docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz +docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz + +# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取 +# docker pull bigcodebench/bigcodebench-evaluate:latest +# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF' +# FROM bigcodebench/bigcodebench-evaluate:latest +# ENTRYPOINT [] +# CMD ["tail", "-f", "/dev/null"] +# EOF +# docker pull python:3.11-slim +``` + +验证： + +```bash +docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim' +``` + +--- + +## 4. `bash/run.py` 完整参数 | 参数 | 默认值 | 说明 | |------|--------|------| @@ -106,7 +262,7 @@ python bash/run.py \ --- -## 4. 套件与 Benchmark 覆盖 +## 5. 套件与 Benchmark 覆盖 ### 4.1 `full` 全量套件 @@ -160,7 +316,7 @@ python bash/run.py \ --- -## 5. Multi-run 配置 +## 6. Multi-run 配置 以下 benchmark 默认会重复跑多次再取平均，次数在 `bash/run.py` 的 `MULTI_RUN_CONFIG` 中定义： @@ -174,7 +330,7 @@ python bash/run.py \ --- -## 6. 输出目录与统计 +## 7. 输出目录与统计 ### 6.1 目录结构 @@ -231,7 +387,7 @@ for f in sorted(glob.glob('output/{folder_name}/*/seed_*/reports/*/*.json')): --- -## 7. 快速开始 +## 8. 快速开始 ### 7.1 安装依赖 @@ -275,7 +431,7 @@ python bash/run.py --help --- -## 8. 设计说明 +## 9. 设计说明 - **模型无关**：`run.py` 通过 `--model` / `--api-url` 接入任意 OpenAI-compatible 服务。 - **配置可覆盖**：YAML 配置 + 命令行参数，未配置的 benchmark 自动使用默认生成参数。 @@ -285,7 +441,7 @@ python bash/run.py --help --- -## 9. 相关文档 +## 10. 相关文档 - 详细使用手册：`myread.md` - Docker 构建说明：`DOCKER_BUILD.md` diff --git a/bash/case/DP4-flash-int8-no-thinking-full.sh b/bash/case/DP4-flash-int8-no-thinking-full.sh new file mode 100644 index 0000000..714b298 --- /dev/null +++ b/bash/case/DP4-flash-int8-no-thinking-full.sh @@ -0,0 +1 @@ +python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking diff --git a/bash/case/DP4-flash-int8-thinking-official.sh b/bash/case/DP4-flash-int8-thinking-official.sh new file mode 100644 index 0000000..0e5d7ac --- /dev/null +++ b/bash/case/DP4-flash-int8-thinking-official.sh @@ -0,0 +1,6 @@ +python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking +python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking --max-tokens-add 0 +python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking --max-tokens-add 16384 +python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking --max-tokens-add 32768 +python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking --max-tokens-add 65536 +python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking --max-tokens-add 131072 diff --git a/bash/collect_results.py b/bash/collect_results.py index f03a850..487215b 100644 --- a/bash/collect_results.py +++ b/bash/collect_results.py @@ -348,8 +348,9 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): tpot_p99 = percentile(tpots, 99) # The actual sample count we just rebuilt from raw predictions is more # reliable than the (possibly reset) report summary's n_samples. - if sample_indexes: - n_samples_unique = len(sample_indexes) + # For multi-seed / multi-run benchmarks we report the total number of + # evaluated predictions (all seeds/runs) rather than unique problem IDs. + n_samples_unique = len(latencies) elif summary0: # Fallback to report summary if raw predictions are unavailable latency_mean = summary0.get('latency', {}).get('mean', np.nan) @@ -552,11 +553,15 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None, # Add total row total_score = df['得分'].mean() total_time = df['实测时间(h)'].sum() + total_samples = df['总样本数'].sum() if '总样本数' in df.columns else np.nan + total_tokens = df['累计总tokens'].sum() if '累计总tokens' in df.columns else np.nan total_row = { '分类': '总计', 'Benchmark': '', '得分': round(total_score, 4), '实测时间(h)': round(total_time, 4), + '总样本数': total_samples if not np.isnan(total_samples) else np.nan, + '累计总tokens': total_tokens if not np.isnan(total_tokens) else np.nan, } for col in OUTPUT_COLUMNS: if col not in total_row: diff --git a/bash/run.py b/bash/run.py index 47e342c..7d20fa7 100644 --- a/bash/run.py +++ b/bash/run.py @@ -151,8 +151,12 @@ SUITES = { 'official': { 'multi': ['aime25', 'aime26', 'live_code_bench'], 'single': [ - 'hle', 'mmlu_pro', 'gpqa_diamond', 'longbench_v2', - # 'swe_bench_verified', + + 'hle', + 'mmlu_pro', + 'gpqa_diamond', + 'longbench_v2', + 'swe_bench_verified', ], 'agent': [ 'tau2_bench' diff --git a/bash/test.py b/bash/test.py deleted file mode 100644 index 6c7c285..0000000 --- a/bash/test.py +++ /dev/null @@ -1,659 +0,0 @@ -#!/usr/bin/env python3 -""" -Unified benchmark runner for EvalScope. - -A single entry point for lite / mid / full / group1 / group2 / group3 evaluations. -All tunable parameters can be controlled via command-line arguments. - -Examples: - # Full evaluation (all benchmarks, multi-run for stability) - python bash/run.py \ - --model DeepSeek-V4-Flash-Int8 \ - --api-url http://localhost:30000/v1 \ - --dataset-dir /data1/sora/evalscope \ - --output-dir /data1/sora/evalscope/output \ - --suite full \ - --limit none - - # Lite smoke test (~5h with full samples) - python bash/run.py --suite lite --limit none - - # Run only selected benchmarks - python bash/run.py --datasets aime24,gsm8k,arc --limit 20 - - # Custom judge model - python bash/run.py \ - --judge-model deepseek-v4-pro \ - --judge-api-url https://api.deepseek.com/v1 \ - --judge-api-key sk-xxx -""" - -import argparse -import json -import sys -import time -from copy import deepcopy -from pathlib import Path - -import yaml - -from evalscope import run_task, TaskConfig -from evalscope.api.agent import NativeAgentConfig -from evalscope.config import SandboxTaskConfig - -SCRIPT_DIR = Path(__file__).parent.resolve() -PROJECT_ROOT = SCRIPT_DIR.parent - -# Make collect_results importable -sys.path.insert(0, str(SCRIPT_DIR)) -import collect_results as collect_results_module -import perf_backup as perf_backup_module - -# ============================================================ -# Default configuration (override via CLI) -# ============================================================ - -DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8' -DEFAULT_API_URL = 'http://localhost:30000/v1' -DEFAULT_DATASET_DIR = str(PROJECT_ROOT) -DEFAULT_OUTPUT_DIR = str(PROJECT_ROOT / 'output') -DEFAULT_CONFIG = str(PROJECT_ROOT / 'config' / 'dpv4-int8_nothinking.yaml') -DEFAULT_TOKENIZER_PATH = '/data1/models/DeepSeek-V4-Flash-INT8' -DEFAULT_LIMIT = None -DEFAULT_SEED = 42 -DEFAULT_BATCH_SIZE = 4 -DEFAULT_ENABLE_THINKING = False - -DEFAULT_JUDGE_MODEL = 'DeepSeek/DeepSeek-V4-Pro' -DEFAULT_JUDGE_API_URL = 'https://api.vectron.meta-stone.com/v1' -DEFAULT_JUDGE_API_KEY = 'sk-dbd8a665f7634081b87ec409c7636500' -DEFAULT_JUDGE_MAX_TOKENS = 10240 - -# 长文本 middle-truncation 上限（token 数）。当前默认 128k。 -DEFAULT_TRUNCATION_TOKENS = 32768 * 4 - -# ============================================================ -# Benchmark suites -# ============================================================ - -# 多次采样配置：总样本数控制在 ~400-500 -MULTI_RUN_CONFIG = { - 'aime24': 12, - 'aime25': 12, - 'aime26': 12, - 'hmmt26': 12, - 'live_code_bench': 5, - 'imo_answerbench': 4, - 'humaneval': 3, - 'gpqa_diamond': 2, -} - -# 能力域完整列表 -ALL_MULTI_RUN = [ - # 'humaneval', 'live_code_bench', - # 'aime24', 'aime25', 'aime26', 'hmmt26', - # 'imo_answerbench', 'gpqa_diamond', -] - - -ALL_SINGLE_RUN = [ - # 'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa', - # 'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro', - # 'simple_qa', 'trivia_qa', 'winogrande', - # 'openai_mrcr', 'longbench_v2', - 'swe_bench_verified', 'swe_bench_pro', -] -ALL_AGENT = [ - # 'tau2_bench', 'general_fc' - ] - -# 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡： -# Group1: ~61h | Group2: ~62h | Group3: ~55h -SUITES = { - 'full': { - 'multi': ALL_MULTI_RUN, - 'single': ALL_SINGLE_RUN, - 'agent': ALL_AGENT, - }, - 'lite': { - 'multi': ['aime24', 'humaneval'], - 'single': ['gsm8k', 'arc', 'longbench_v2'], - 'agent': ['general_fc'], - }, - 'mid': { - 'multi': ['aime24', 'humaneval'], - 'single': [ - 'live_code_bench', 'bigcodebench', 'competition_math', 'gsm8k', - 'gpqa_diamond', 'mmlu_pro', 'simple_qa', 'longbench_v2', 'openai_mrcr', - ], - 'agent': ['general_fc', 'tau2_bench'], - }, - # 多机组分组，基于 CSV 实测完整时间（已含 multi-run）平衡： - # Group1: ~22.7h | Group2: ~25.6h | Group3: ~27.0h | 合计 ~75.2h - 'group1': { - 'multi': ['live_code_bench', 'aime24', 'aime25', 'aime26', 'hmmt26', 'imo_answerbench', 'humaneval'], - 'single': ['bigcodebench', 'competition_math', 'gsm8k', 'drop', 'arc', 'hellaswag', 'winogrande'], - 'agent': [], - }, - 'group2': { - 'multi': [], - 'single': ['hle', 'mmlu_pro', 'trivia_qa'], - 'agent': [], - }, - 'group3': { - 'multi': ['gpqa_diamond'], - 'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'], - 'agent': ['tau2_bench', 'general_fc'], - }, -} - -# ============================================================ -# Fixed configuration -# ============================================================ - -MATH_DATASETS = { - 'aime24', 'aime25', 'aime26', 'hmmt26', - 'gsm8k', 'competition_math', 'imo_answerbench', -} - -MATH_PROMPT_TEMPLATE = ( - "{question}\n" - "Please reason step by step, and put your final answer within \\boxed{{}}." -) - -SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'swe_bench_verified', 'swe_bench_pro'} -SANDBOX_CONFIGS = { - 'bigcodebench': { - 'image': 'bigcodebench-sandbox:latest', - 'working_dir': '/tmp', - 'tools_config': { - 'shell_executor': {}, - 'python_executor': {} - } - }, - 'humaneval': { - 'image': 'python:3.11-slim', - 'tools_config': { - 'shell_executor': {}, - 'python_executor': {} - } - }, - 'swe_bench_verified': { - 'image': 'swe-bench-sandbox:latest', - 'working_dir': '/tmp', - 'tools_config': { - 'shell_executor': {}, - 'python_executor': {} - } - }, - 'swe_bench_pro': { - 'image': 'swe-bench-sandbox:latest', - 'working_dir': '/tmp', - 'tools_config': { - 'shell_executor': {}, - 'python_executor': {} - } - }, -} - - -# ============================================================ -# CLI parser -# ============================================================ - -def build_parser(): - parser = argparse.ArgumentParser( - description='Unified EvalScope benchmark runner', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog='Suites: full, lite, mid, group1, group2, group3', - ) - - # Model / API - parser.add_argument('--model', default=DEFAULT_MODEL, - help='Served model name (default: %(default)s)') - parser.add_argument('--api-url', default=DEFAULT_API_URL, - help='OpenAI-compatible API URL (default: %(default)s)') - - # Paths - parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR, - help='Parent directory containing datasets/ subdir (default: %(default)s)') - parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR, - help='Output root directory (default: %(default)s)') - parser.add_argument('--config', default=DEFAULT_CONFIG, - help='YAML config path (default: %(default)s)') - parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH, - help='Local tokenizer path for middle-truncation (default: %(default)s)') - - # Run control - parser.add_argument('--suite', default='full', choices=list(SUITES.keys()), - help='Benchmark suite to run (default: %(default)s)') - parser.add_argument('--datasets', '--benchmarks', dest='datasets', default=None, - help='Override suite with comma-separated benchmark names, e.g. aime24,gsm8k') - parser.add_argument('--exclude', default=None, - help='Comma-separated benchmarks to exclude from the chosen suite') - parser.add_argument('--limit', default=None, - help='Max samples per benchmark; "none"/"all" for no limit (default: none)') - parser.add_argument('--seed', type=int, default=DEFAULT_SEED, - help='Random seed (default: %(default)s)') - parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE, - help='Evaluation batch size (default: %(default)s)') - - # Decoding / thinking - parser.add_argument('--thinking', action='store_true', default=None, - help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)') - parser.add_argument('--no-thinking', dest='thinking', action='store_false', - help='Disable thinking mode (default)') - - # Judge model - parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL, - help='Judge model name (default: %(default)s)') - parser.add_argument('--judge-api-url', default=DEFAULT_JUDGE_API_URL, - help='Judge model API URL (default: %(default)s)') - parser.add_argument('--judge-api-key', default=DEFAULT_JUDGE_API_KEY, - help='Judge model API key') - parser.add_argument('--judge-max-tokens', type=int, default=DEFAULT_JUDGE_MAX_TOKENS, - help='Judge model max_tokens (default: %(default)s)') - - # Truncation - parser.add_argument('--truncation-tokens', type=int, default=DEFAULT_TRUNCATION_TOKENS, - help='Middle-truncation token budget for long-context benchmarks (default: %(default)s)') - - # Result collection - parser.add_argument('--no-summary', dest='write_summary', action='store_false', - help='Skip writing summary Excel/CSV after each benchmark') - parser.add_argument('--summary-name', default=None, - help='Output summary file name (without extension); defaults to safe model name') - - return parser - - -# ============================================================ -# Middle-truncation helpers -# ============================================================ - -_TOKENIZER = None - - -def get_tokenizer(tokenizer_path: str): - global _TOKENIZER - if _TOKENIZER is None: - from transformers import AutoTokenizer - try: - _TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) - except Exception: - _TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True) - return _TOKENIZER - - -def truncate_middle(text: str, max_tokens: int, tokenizer_path: str) -> str: - if max_tokens <= 0: - return text - tokenizer = get_tokenizer(tokenizer_path) - token_ids = tokenizer.encode(text, add_special_tokens=False) - if len(token_ids) <= max_tokens: - return text - keep_head = max_tokens // 2 - keep_tail = max_tokens - keep_head - truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:] - return tokenizer.decode(truncated_ids, skip_special_tokens=True) - - -def _patch_adapters_for_truncation(tokenizer_path: str, truncation_tokens: int): - from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter - from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter - - _orig_longbench_format = LongBenchV2Adapter.format_prompt_template - - def _patched_longbench_format(self, sample): - if sample.metadata and 'context' in sample.metadata: - sample.metadata['context'] = truncate_middle(sample.metadata['context'], truncation_tokens, tokenizer_path) - return _orig_longbench_format(self, sample) - - LongBenchV2Adapter.format_prompt_template = _patched_longbench_format - - _orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample - - def _patched_mrcr_record(self, record): - per_msg_max_tok = 8192 - if 'prompt' in record: - try: - prompt_data = json.loads(record['prompt']) - if not isinstance(prompt_data, list) or len(prompt_data) == 0: - return _orig_mrcr_record(self, record) - - tokenizer = get_tokenizer(tokenizer_path) - total_tok = sum( - len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False)) - for msg in prompt_data - ) - if total_tok <= truncation_tokens: - return _orig_mrcr_record(self, record) - - desired_idx = record.get('desired_msg_index', 0) - if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data): - desired_idx = 0 - - n = len(prompt_data) - keep = set() - keep.update(range(min(2, n))) - keep.update(range(max(0, n - 2), n)) - window = 2 - keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1))) - keep = sorted(keep) - - new_prompt = [] - for idx in keep: - msg = prompt_data[idx] - if isinstance(msg, dict): - msg = dict(msg) - content = msg.get('content', '') - if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok: - msg['content'] = truncate_middle(content, per_msg_max_tok, tokenizer_path) - new_prompt.append(msg) - - record = dict(record) - record['prompt'] = json.dumps(new_prompt) - except (json.JSONDecodeError, TypeError): - pass - return _orig_mrcr_record(self, record) - - OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record - - -# ============================================================ -# Helpers -# ============================================================ - -def load_dataset_configs(config_path: str): - if not Path(config_path).exists(): - raise FileNotFoundError(f'Config file not found: {config_path}') - with open(config_path, 'r', encoding='utf-8') as f: - return yaml.safe_load(f) - - -def configure_thinking(generation_config: dict, enable: bool) -> dict: - extra_body = generation_config.get('extra_body', {}) - chat_template_kwargs = extra_body.get('chat_template_kwargs', {}) - if enable: - chat_template_kwargs['thinking'] = True - else: - chat_template_kwargs.pop('thinking', None) - if chat_template_kwargs: - extra_body['chat_template_kwargs'] = chat_template_kwargs - if extra_body: - generation_config['extra_body'] = extra_body - return generation_config - - -def build_agent_config(agent_cfg: dict) -> NativeAgentConfig: - agent_cfg = deepcopy(agent_cfg or {}) - known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'} - kwargs = agent_cfg.pop('kwargs', {}) - for key in list(agent_cfg.keys()): - if key not in known_fields: - kwargs[key] = agent_cfg.pop(key) - if kwargs: - agent_cfg['kwargs'] = kwargs - return NativeAgentConfig(**agent_cfg) - - -def build_task_config( - dataset_name: str, - ds_cfg: dict, - batch_size: int, - enable_thinking: bool, - seed: int, - limit, - output_dir: str, - model: str, - api_url: str, - dataset_dir: str, - judge_model_args: dict, - run_idx: int = 0, -) -> TaskConfig: - if run_idx > 0: - work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}' - else: - work_dir = Path(output_dir) / dataset_name / f'seed_{seed}' - work_dir.mkdir(parents=True, exist_ok=True) - work_dir = str(work_dir) - - generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking) - dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) - dataset_args.setdefault('shuffle', True) - - if dataset_name in MATH_DATASETS: - dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE - - dataset_args_dict = {dataset_name: dataset_args} - - agent_config = None - if 'agent_config' in ds_cfg: - agent_config = build_agent_config(ds_cfg['agent_config']) - - return TaskConfig( - model=model, - api_url=api_url, - eval_type='openai_api', - dataset_dir=dataset_dir, - judge_model_args=judge_model_args, - seed=seed, - limit=limit, - collect_perf=True, - no_timestamp=True, - work_dir=work_dir, - use_cache=work_dir, - datasets=[dataset_name], - generation_config=generation_config, - dataset_args=dataset_args_dict, - agent_config=agent_config, - eval_batch_size=batch_size, - sandbox=SandboxTaskConfig( - enabled=True, - engine='docker', - default_config=SANDBOX_CONFIGS.get(dataset_name, { - 'image': 'python:3.11-slim', - 'tools_config': { - 'shell_executor': {}, - 'python_executor': {} - } - }) - ) if dataset_name in SANDBOX_DATASETS else None, - ) - - -# ============================================================ -# Main -# ============================================================ - -def write_summary(output_dir: str, model_name: str, summary_name: str): - """Re-aggregate results across all benchmarks under output_dir.""" - try: - excel_output_dir = PROJECT_ROOT / 'results' - collect_results_module.collect_all( - Path(output_dir), model_name, summary_name, - excel_output_dir=excel_output_dir, - ) - except Exception as e: - print(f'WARNING: failed to write summary: {e}') - - -def backup_after_run(output_dir: str, benchmark: str, model_name: str, - work_dir: Path): - """Snapshot the just-finished run's perf summary and predictions.""" - try: - report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' - perf_backup_module.backup_perf_stats( - Path(output_dir), benchmark, model_name, report_json, - ) - predictions_dir = work_dir / 'predictions' / model_name - perf_backup_module.archive_predictions( - Path(output_dir), benchmark, model_name, predictions_dir, - ) - except Exception as e: - print(f'WARNING: perf backup for {benchmark} failed: {e}') - - -def restore_before_run(output_dir: str, benchmark: str, model_name: str, - work_dir: Path) -> bool: - """If durable backups exist for a benchmark/model, materialise them into - the new ``work_dir`` before evalscope starts so the next run resumes from - the larger historical state instead of overwriting it. - """ - try: - predictions_dir = work_dir / 'predictions' / model_name - report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' - restored = perf_backup_module.restore_from_backup( - Path(output_dir), benchmark, model_name, - predictions_dir, report_json, - ) - if restored: - print(f'Restored {benchmark}/{model_name} from backup before run') - return restored - except Exception as e: - print(f'WARNING: perf restore for {benchmark} failed: {e}') - return False - - -def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, model_name: str, summary_name: str): - """Run a benchmark task and optionally refresh the summary table. - - After a successful ``run_task()`` we snapshot the cumulative - ``perf_metrics.summary`` and the per-sample predictions into the durable - backup files maintained by ``perf_backup.py`` so checkpoint restarts can - recover them. - """ - dataset_name = task_cfg.datasets[0] - work_dir = Path(task_cfg.work_dir) - restore_before_run(output_dir, dataset_name, model_name, work_dir) - - start_ts = time.monotonic() - try: - run_task(task_cfg) - finally: - elapsed = time.monotonic() - start_ts - perf_backup_module.record_active_time(output_dir, dataset_name, model_name, elapsed) - print(f'Active time for {dataset_name}: {elapsed:.1f}s (total accumulated)') - - backup_after_run(output_dir, dataset_name, model_name, work_dir) - if write_summary_flag: - write_summary(output_dir, model_name, summary_name) - - -def main(): - parser = build_parser() - args = parser.parse_args() - - # Resolve limit - limit = args.limit - if limit is not None: - if str(limit).lower() in ('none', 'all'): - limit = None - else: - limit = int(limit) - - enable_thinking = DEFAULT_ENABLE_THINKING if args.thinking is None else args.thinking - - # Resolve suite or custom datasets - if args.datasets: - custom = [d.strip() for d in args.datasets.split(',') if d.strip()] - multi_run = [d for d in custom if d in MULTI_RUN_CONFIG] - single_run = [d for d in custom if d not in MULTI_RUN_CONFIG] - agent = [d for d in custom if d in ALL_AGENT] - single_run = [d for d in single_run if d not in ALL_AGENT] - else: - suite = SUITES[args.suite] - multi_run = list(suite['multi']) - single_run = list(suite['single']) - agent = list(suite['agent']) - - # Apply --exclude - if args.exclude: - exclude = {d.strip() for d in args.exclude.split(',') if d.strip()} - multi_run = [d for d in multi_run if d not in exclude] - single_run = [d for d in single_run if d not in exclude] - agent = [d for d in agent if d not in exclude] - - judge_model_args = { - 'model_id': args.judge_model, - 'api_url': args.judge_api_url, - 'api_key': args.judge_api_key, - 'eval_type': 'openai_api', - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': args.judge_max_tokens, - }, - } - - truncation_tokens = args.truncation_tokens - _patch_adapters_for_truncation(args.tokenizer_path, truncation_tokens) - - dataset_configs = load_dataset_configs(args.config) - - print('=' * 60) - print(f'Config: {args.config}') - print(f'Model: {args.model}') - print(f'API URL: {args.api_url}') - print(f'Dataset Dir: {args.dataset_dir}') - print(f'Output Dir: {args.output_dir}') - print(f'Suite: {args.suite}') - print(f'Limit: {limit if limit is not None else "ALL"}') - print(f'Thinking: {enable_thinking}') - print(f'Seed: {args.seed}') - print(f'Batch Size: {args.batch_size}') - print(f'Tokenizer Path: {args.tokenizer_path}') - print(f'Truncation Tokens: {truncation_tokens}') - print(f'Multi-run datasets: {multi_run}') - print(f'Single-run datasets: {single_run}') - print(f'Agent datasets: {agent}') - print(f'Write summary: {args.write_summary}') - print('=' * 60) - - def run_one(dataset_name, run_idx=0): - ds_cfg = dataset_configs[dataset_name] - task_cfg = build_task_config( - dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, - run_idx=run_idx, - ) - try: - run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, args.summary_name) - except Exception as e: - print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}') - - for dataset_name in multi_run: - if dataset_name not in dataset_configs: - print(f'WARNING: {dataset_name} not in YAML config, skipping') - continue - num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) - for run_idx in range(num_runs): - print(f"\n{'='*60}") - print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})') - print(f"{'='*60}") - run_one(dataset_name, run_idx=run_idx) - - for dataset_name in single_run: - if dataset_name not in dataset_configs: - print(f'WARNING: {dataset_name} not in YAML config, skipping') - continue - print(f"\n{'='*60}") - print(f'Running: {dataset_name} (seed={args.seed})') - print(f"{'='*60}") - run_one(dataset_name) - - for dataset_name in agent: - if dataset_name not in dataset_configs: - print(f'WARNING: {dataset_name} not in YAML config, skipping') - continue - print(f"\n{'='*60}") - print(f'Running: {dataset_name} (seed={args.seed})') - print(f"{'='*60}") - run_one(dataset_name) - - if args.write_summary: - write_summary(args.output_dir, args.model, args.summary_name) - print('\nAll benchmarks done!') - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/bash/test_e2e.py b/bash/test_e2e.py deleted file mode 100644 index 896c5f8..0000000 --- a/bash/test_e2e.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -"""End-to-end offline test for bash/run.py + collect_results + perf_backup. - -This script does NOT call any model API. It uses an existing benchmark -directory (aime24/seed_42) and walks through every code path that the real -``run.py`` would touch: - -1. backup_perf_stats — confirm a perf summary snapshot is written -2. archive_predictions — confirm per-sample records are deduplicated into - the archive -3. restore_from_backup — confirm a fresh work_dir can be rehydrated from - the backup files -4. collect_results aggregation — confirm correct metrics are produced -5. Whitelist mode — confirm only the requested benchmarks appear in the - summary -6. hle_low alias — confirm hle_low directory maps to canonical "hle" -7. Reset recovery — confirm that even after wiping the predictions file - and the report summary, the archive+backup still recover the metrics - -Run from the project root: - - python bash/test_e2e.py -""" - -import json -import shutil -import sys -import tempfile -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) - -import collect_results as cr # noqa: E402 -import perf_backup as pb # noqa: E402 - - -PROJECT_ROOT = Path('/data1/sora/evalscope') -SOURCE_OUTPUT = PROJECT_ROOT / 'output' - - -def banner(msg): - print('\n' + '=' * 60) - print(f' {msg}') - print('=' * 60) - - -def assert_close(actual, expected, name, tol=1e-3): - if abs(actual - expected) > tol: - raise AssertionError(f'{name}: expected ~{expected}, got {actual}') - print(f' OK {name}={actual:.4f}') - - -def main(): - banner('Setting up isolated test output dir') - test_root = PROJECT_ROOT / 'output_e2e_test' - if test_root.exists(): - shutil.rmtree(test_root) - test_root.mkdir(parents=True) - - # Copy the canonical aime24 data into the isolated dir - src = SOURCE_OUTPUT / 'aime24' / 'seed_42' - dst = test_root / 'aime24' / 'seed_42' - shutil.copytree(src, dst) - - # And copy hle_low as hle_low under the test dir (so we exercise the - # alias code path). - hle_src = SOURCE_OUTPUT / 'hle_low' / 'seed_42' - hle_dst = test_root / 'hle_low' / 'seed_42' - if hle_src.exists(): - shutil.copytree(hle_src, hle_dst) - - benchmark = 'aime24' - model_name = 'DeepSeek-V4-Flash-Int8' - - banner('1) backup_perf_stats — snapshot summary') - report_json = dst / 'reports' / model_name / f'{benchmark}.json' - backup_path = pb.backup_perf_stats(test_root, benchmark, model_name, report_json) - assert backup_path.exists(), 'backup file was not created' - payload = json.loads(backup_path.read_text()) - assert payload['n_samples'] == 30, f'expected 30 samples, got {payload["n_samples"]}' - print(f' OK backup written: {backup_path}, n_samples={payload["n_samples"]}') - - banner('2) archive_predictions — deduplicate into archive') - preds = dst / 'predictions' / model_name - archive_path = pb.archive_predictions(test_root, benchmark, model_name, preds) - assert archive_path.exists(), 'archive file was not created' - n_lines = sum(1 for line in archive_path.read_text().splitlines() if line.strip()) - assert n_lines == 30, f'expected 30 archived samples, got {n_lines}' - print(f' OK archive written: {archive_path}, samples={n_lines}') - - banner('3) restore_from_backup — rehydrate a fresh work_dir') - fresh_dir = test_root / 'fresh_work' / 'seed_42' - fresh_report = fresh_dir / 'reports' / model_name / f'{benchmark}.json' - fresh_preds = fresh_dir / 'predictions' / model_name - # Pre-create empty structure to mimic what eval_task would do - fresh_report.parent.mkdir(parents=True, exist_ok=True) - fresh_report.write_text(json.dumps({ - 'name': f'{model_name}@{benchmark}', - 'perf_metrics': {'summary': {'n_samples': 1}}, - })) - restored = pb.restore_from_backup( - test_root, benchmark, model_name, fresh_preds, fresh_report, - ) - assert restored, 'restore_from_backup returned False' - restored_report = json.loads(fresh_report.read_text()) - assert restored_report['perf_metrics']['summary']['n_samples'] == 30, \ - f'restored n_samples={restored_report["perf_metrics"]["summary"]["n_samples"]}' - print(' OK restored report n_samples=30 (matches backup)') - - banner('4) collect_results — aggregate metrics') - csv, xlsx = cr.eval_benchmark([benchmark], test_root, model_name) - import pandas as pd - df = pd.read_csv(csv) - aime24 = df[df['Benchmark'] == 'aime24'].iloc[0] - # Single seed_42 run, so values match the report directly (no multi-run - # averaging). The full-summary CSV averages across run_1..run_11. - assert_close(aime24['得分'], 0.6333, 'aime24 score (single seed)') - assert_close(aime24['实测时间(h)'], 0.1469, 'aime24 duration (h)') - assert_close(aime24['总样本数'], 30, 'aime24 sample count') - assert_close(aime24['延迟_mean(s)'], 35.24971, 'aime24 latency mean') - assert_close(aime24['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean') - assert_close(aime24['TPOT_mean(s)'], 0.02165, 'aime24 TPOT mean') - assert_close(aime24['输入tokens_mean'], 119.33, 'aime24 input_tokens_mean') - assert_close(aime24['输出tokens_mean'], 1609.8, 'aime24 output_tokens_mean') - - banner('5) Whitelist mode — only the requested benchmarks appear') - csv2, _ = cr.eval_benchmark([benchmark], test_root, model_name) - df2 = pd.read_csv(csv2) - assert df2['Benchmark'].dropna().tolist() == ['aime24'], \ - f'whitelist did not limit benchmarks: {df2["Benchmark"].tolist()}' - print(' OK summary only contains aime24 + total') - - banner('6) hle_low → hle alias') - if hle_src.exists(): - csv3, _ = cr.eval_benchmark(['hle'], test_root, model_name) - df3 = pd.read_csv(csv3) - assert 'hle' in df3['Benchmark'].tolist(), \ - f'hle canonical name missing from: {df3["Benchmark"].tolist()}' - assert 'hle_low' not in df3['Benchmark'].tolist(), \ - f'hle_low should be aliased away: {df3["Benchmark"].tolist()}' - print(' OK hle_low directory maps to canonical "hle"') - - banner('7) Reset recovery — wipe report+predictions, expect archive to restore') - # Wipe the current predictions - shutil.rmtree(preds) - # Reset report.json - data = json.loads(report_json.read_text()) - data['perf_metrics']['summary']['n_samples'] = 1 - data['perf_metrics']['summary']['latency']['mean'] = 0.0 - report_json.write_text(json.dumps(data, indent=2)) - - csv4, _ = cr.eval_benchmark([benchmark], test_root, model_name) - df4 = pd.read_csv(csv4) - aime24_after = df4[df4['Benchmark'] == 'aime24'].iloc[0] - assert_close(aime24_after['得分'], 0.6333, 'aime24 score (post-reset)') - assert_close(aime24_after['总样本数'], 30, 'aime24 sample count (post-reset)') - assert_close(aime24_after['延迟_mean(s)'], 35.24971, 'aime24 latency mean (post-reset)') - assert_close(aime24_after['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean (post-reset)') - print(' OK archive fully recovered metrics after predictions wipe') - - banner('8) Backup monotonicity — new backup must not overwrite a larger one') - # We have a backup with n_samples=30. Now write a fresh report with - # n_samples=2 and call backup_perf_stats again — the backup should NOT - # be overwritten. - data = json.loads(report_json.read_text()) - data['perf_metrics']['summary']['n_samples'] = 2 - report_json.write_text(json.dumps(data, indent=2)) - pb.backup_perf_stats(test_root, benchmark, model_name, report_json) - payload2 = json.loads(backup_path.read_text()) - assert payload2['n_samples'] == 30, \ - f'backup was clobbered: n_samples now {payload2["n_samples"]}' - print(' OK backup preserved n_samples=30 even after a reset run') - - banner('All checks passed') - print(f'Test artifacts under {test_root} (kept for inspection)') - - -if __name__ == '__main__': - main() diff --git a/evalscope/evalscope/benchmarks/swe_bench/utils.py b/evalscope/evalscope/benchmarks/swe_bench/utils.py index 5e7de63..4e49a76 100644 --- a/evalscope/evalscope/benchmarks/swe_bench/utils.py +++ b/evalscope/evalscope/benchmarks/swe_bench/utils.py @@ -101,6 +101,28 @@ def eval_instance( log_dir = Path(log_dir) / 'swebench_log' / instance_id log_dir.mkdir(parents=True, exist_ok=True) + + # Resume: skip container evaluation if test output already exists. + test_output_path = log_dir / LOG_TEST_OUTPUT + if test_output_path.exists(): + logger.info(f'Found existing test output for {instance_id}, skip container evaluation and reuse log.') + try: + report = get_eval_report( + test_spec=test_spec, + prediction=pred, + test_log_path=test_output_path, + include_tests_status=True, + ) + logger.info(f'report: {report}\n' + f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}") + return { + 'completed': True, + 'resolved': report.get(instance_id, {}).get('resolved', False), + 'report': report, + } + except Exception as e: + logger.warning(f'Failed to reuse existing test output for {instance_id}: {e}. Will re-evaluate.') + logger.info(f'Starting evaluation for {instance_id} in log dir {log_dir}...') client = DockerClient.from_env() diff --git a/myread.md b/myread.md index 64b4b0b..7b78069 100644 --- a/myread.md +++ b/myread.md @@ -159,12 +159,65 @@ docker images | grep evalscope-complete-py312 | `python:3.11-slim` | 通用代码执行环境 | | `swebench/sweb.eval.x86_64.*` | SWE-bench 每个样本一个实例镜像 | -#### 2.4.1 配置 Docker（国内镜像加速 + 大容量存储路径） +根据是否跑 SWE-bench，选择下面两种 Docker 配置方式之一。 -SWE-bench 镜像数量多、体积大，建议把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘： +#### 2.4.1 方式 A：不测 SWE-bench 的 Docker 配置 + +如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像： ```bash -# 1. 停止 Docker +# 1. 暂停服务 +sudo systemctl stop docker.socket +sudo systemctl stop docker + +# 2. 配置 Docker 国内镜像加速 +sudo tee /etc/docker/daemon.json <<-'EOF' +{ + "registry-mirrors": [ + "https://docker.m.daocloud.io", + "https://docker.1ms.run", + "https://hub.rat.dev", + "https://docker.1panel.live", + "https://dockerproxy.com", + "https://hub-mirror.c.163.com", + "https://mirror.baidubce.com", + "https://docker.mirrors.ustc.edu.cn", + "https://docker.mirrors.sjtug.sjtu.edu.cn", + "https://docker.nju.edu.cn", + "https://docker.mirrors.tuna.tsinghua.edu.cn" + ] +} +EOF + +# 3. 重启 Docker +sudo systemctl start docker + +# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖） +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz + +# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须） +docker pull bigcodebench/bigcodebench-evaluate:latest +docker build -t bigcodebench-sandbox:latest -f - . <<'EOF' +FROM bigcodebench/bigcodebench-evaluate:latest +ENTRYPOINT [] +CMD ["tail", "-f", "/dev/null"] +EOF + +docker pull python:3.11-slim +``` + +验证： + +```bash +docker images | grep -E 'evalscope-complete-py312|bigcodebench|python:3.11-slim' +``` + +#### 2.4.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径） + +SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘，否则默认盘很快会被占满： + +```bash +# 1. 暂停服务 sudo systemctl stop docker.socket sudo systemctl stop docker sudo systemctl stop containerd @@ -173,7 +226,7 @@ sudo systemctl stop containerd mkdir -p /data1/sora/evalscope/docker/images mkdir -p /data1/sora/evalscope/docker/containerd -# 3. 配置国内镜像加速（多填几个，自动轮询） +# 3. 配置国内镜像加速 + 大容量 data-root sudo tee /etc/docker/daemon.json <<-'EOF' { "data-root": "/data1/sora/evalscope/docker/images", @@ -203,22 +256,18 @@ EOF sudo systemctl reset-failed docker.service sudo systemctl start containerd sudo systemctl start docker -``` -#### 2.4.2 拉取/构建代码执行镜像 +# 6. 加载 evalscope 环境镜像 +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz -```bash -# bigcodebench 官方镜像 +# 7. 准备代码执行 sandbox 镜像 docker pull bigcodebench/bigcodebench-evaluate:latest - -# 构建守护态 sandbox docker build -t bigcodebench-sandbox:latest -f - . <<'EOF' FROM bigcodebench/bigcodebench-evaluate:latest ENTRYPOINT [] CMD ["tail", "-f", "/dev/null"] EOF -# 通用 Python 执行环境 docker pull python:3.11-slim ``` @@ -267,25 +316,42 @@ python bash/pull_swe_bench_images.py \ > 2. 降低 `--max-workers` 到 1； > 3. 使用 `tmux` 挂后台运行，避免 SSH 断连导致中断。 -#### 2.4.4 SWE-bench 镜像的本地备份与迁移（可选） +#### 2.4.4 SWE-bench 镜像的备份与迁移（可选） -SWE-bench 全量镜像约 500 个、总大小约 **2TB**，无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器，建议用移动硬盘或内网直接拷贝 Docker `data-root`，或使用下面脚本分批导出/导入： +SWE-bench Verified 全量约 500 个镜像、本地实际占用约 **160GB**。迁移到另一台机器有三种方式： -**源机器：打包** +**方式一：从 ModelScope 下载（推荐，已上传）** + +`SoraAmami/swe-bench-verified-images` 已包含分批导出的 `docker save` tar.gz，目标机器直接下载加载即可： ```bash -bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50 -``` +# 下载 +modelscope download \ + --repo-type dataset \ + --local_dir /data1/sora/evalscope/docker/swe_images \ + SoraAmami/swe-bench-verified-images -脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。 - -**目标机器：加载** - -```bash +# 加载 bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images ``` -> 注意：分批 tar 会占用额外磁盘空间（约 2TB），请确保目标盘容量足够。 +**方式二：本地分批导出/导入** + +如果已有本地镜像，可用脚本打包后复制到目标机器： + +```bash +# 源机器打包（每 50 个一批） +bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50 + +# 目标机器加载 +bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images +``` + +**方式三：直接拷贝 Docker data-root** + +适用于内网或移动硬盘，把 `/data1/sora/evalscope/docker/images` 和 `/data1/sora/evalscope/docker/containerd` 完整复制到目标机器相同路径，再启动 Docker。注意这种方式对 Docker 版本和路径要求严格，不如前两种稳定。 + +> 注意：分批 tar 会临时占用额外磁盘空间，加载完成后可删除 tar 包。 --- diff --git a/scripts/Dockerfile.bigcodebench-sandbox b/scripts/Dockerfile.bigcodebench-sandbox new file mode 100644 index 0000000..d8a5c43 --- /dev/null +++ b/scripts/Dockerfile.bigcodebench-sandbox @@ -0,0 +1,58 @@ +# bigcodebench-sandbox +# Minimal sandbox for running BigCodeBench/HumanEval code inside evalscope. +# Based on https://github.com/bigcode-project/bigcodebench/blob/main/Docker/Evaluate.Dockerfile +# but stripped to only the runtime dependencies needed to execute benchmark tests. +# Uses python:3.9-slim because some pinned packages (e.g. numba==0.55.0) do not have +# wheels for newer Python versions. +# Uses Chinese mirrors for apt/pip to speed up builds inside mainland network. + +FROM python:3.9-slim + +# Use Chinese Debian mirror +RUN sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list.d/debian.sources && \ + sed -i 's|http://deb.debian.org/debian-security|http://mirrors.tuna.tsinghua.edu.cn/debian-security|g' /etc/apt/sources.list.d/debian.sources && \ + sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list || true + +# Configure Chinese PyPI mirror +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ + pip config set global.timeout 2000 + +# Install system dependencies needed by the scientific/Python packages +RUN apt-get update && apt-get install -y \ + git \ + g++ \ + python3-tk \ + zip \ + unzip \ + procps \ + r-base \ + libgdal-dev \ + libfreetype6-dev \ + libpng-dev \ + pkg-config \ + python3-dev \ + python3-matplotlib \ + libgl1 \ + && rm -rf /var/lib/apt/lists/* + +# Upgrade pip +RUN pip install --upgrade pip + +# Add a non-root user (matches upstream image conventions) +RUN adduser --disabled-password --gecos "" bigcodebenchuser + +# Copy and install the BigCodeBench evaluation requirements. +# These are the 70+ Python libraries referenced by the benchmark test cases. +COPY requirements-eval.txt /tmp/requirements-eval.txt +RUN pip install -I --timeout 2000 -r /tmp/requirements-eval.txt + +# Ensure a compatible datasets version +RUN pip install datasets==2.17.0 + +WORKDIR /app +RUN chown -R bigcodebenchuser:bigcodebenchuser /app && \ + chmod -R 777 /app + +# For evalscope sandbox: keep container alive +ENTRYPOINT [] +CMD ["tail", "-f", "/dev/null"] diff --git a/scripts/deploy_evalscope.sh b/scripts/deploy_evalscope.sh new file mode 100755 index 0000000..4e66f98 --- /dev/null +++ b/scripts/deploy_evalscope.sh @@ -0,0 +1,292 @@ +#!/bin/bash +# ============================================================ +# EvalScope 完整部署脚本 +# 用法: bash deploy_evalscope.sh [TOKEN] +# 示例: bash deploy_evalscope.sh /data1/sora ms-3d554a39-6e07-496d-8022-0b0ee64a6389 +# ============================================================ + +set -euo pipefail + +# -------------------------------------------------- +# 1. 参数解析 +# -------------------------------------------------- +BASE_DIR="${1:-/data1/sora}" # 基础目录，默认 /data1/sora +DEFAULT_TOKEN="ms-3d554a39-6e07-496d-8022-0b0ee64a6389" +MODELSCOPE_TOKEN="${2:-$DEFAULT_TOKEN}" + +# 派生路径 +EVALSCOPE_DIR="$BASE_DIR/evalscope" +DOCKER_DIR="$EVALSCOPE_DIR/docker" +IMAGES_DIR="$DOCKER_DIR/images" +CONTAINERD_DIR="$DOCKER_DIR/containerd" +SWE_IMAGES_DIR="$DOCKER_DIR/swe_images" +DATASETS_DIR="$EVALSCOPE_DIR/datasets" + +# -------------------------------------------------- +# 2. 安装依赖 +# -------------------------------------------------- +echo "==> 安装 modelscope..." +pip install modelscope + +# -------------------------------------------------- +# 3. 登录 ModelScope +# -------------------------------------------------- +if [ -n "$MODELSCOPE_TOKEN" ]; then + echo "==> 登录 ModelScope..." + modelscope login --token "$MODELSCOPE_TOKEN" +fi + +# -------------------------------------------------- +# 4. 下载数据集 +# -------------------------------------------------- +echo "==> 下载 evalscope 数据集..." +mkdir -p "$DATASETS_DIR" +python3 -c " +from modelscope.hub.snapshot_download import snapshot_download +snapshot_download( + 'SoraAmami/evalscope-datasets', + repo_type='dataset', + cache_dir='$EVALSCOPE_DIR', + local_dir='$DATASETS_DIR' +) +" + +# -------------------------------------------------- +# 5. 下载 Docker 镜像包 +# -------------------------------------------------- +echo "==> 下载 evalscope Docker 镜像..." +mkdir -p "$DOCKER_DIR" +python3 -c " +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-docker', + file_path='evalscope-complete-py312.tar.gz', + local_dir='$DOCKER_DIR' +) +" + +# -------------------------------------------------- +# 6. 配置 Docker 和 containerd 数据目录 +# -------------------------------------------------- +echo "==> 配置 Docker 和 containerd..." + +# 停止服务 +sudo systemctl stop docker.socket 2>/dev/null || true +sudo systemctl stop docker 2>/dev/null || true +sudo systemctl stop containerd 2>/dev/null || true + +# 创建数据目录 +mkdir -p "$IMAGES_DIR" +mkdir -p "$CONTAINERD_DIR" + +# 配置 Docker +echo "==> 写入 Docker 配置..." +sudo tee /etc/docker/daemon.json < 写入 containerd 配置..." +sudo tee /etc/containerd/config.toml </dev/null || true +sudo systemctl start containerd +sudo systemctl start docker + +# -------------------------------------------------- +# 7. 加载 evalscope 环境镜像 +# -------------------------------------------------- +echo "==> 加载 evalscope Docker 镜像..." +docker load -i "$DOCKER_DIR/evalscope-complete-py312.tar.gz" + +# -------------------------------------------------- +# 8. 克隆代码仓库 +# -------------------------------------------------- +echo "==> 克隆 evalscope 代码..." +cd "$EVALSCOPE_DIR" +if [ ! -d "evalstone" ]; then + git clone https://git.meta-stone.net/sora/evalstone.git +fi + +# -------------------------------------------------- +# 9. 准备 sandbox 镜像 +# -------------------------------------------------- +SANDBOX_IMAGES_DIR="$DOCKER_DIR/sandbox_images" +mkdir -p "$SANDBOX_IMAGES_DIR" + +# 函数：带超时拉取镜像，本地已有则跳过 +pull_with_timeout() { + local image="$1" + local timeout_sec="${2:-600}" + if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$image"; then + echo "镜像 $image 已存在，跳过 pull" + return 0 + fi + echo "拉取 $image (超时 ${timeout_sec}s)..." + if timeout "$timeout_sec" docker pull "$image"; then + echo "$image 拉取成功" + return 0 + else + echo "警告：$image 拉取失败或超时" + return 1 + fi +} + +# 优先从 ModelScope 下载预打包的 sandbox 镜像，避免 DockerHub 拉取慢/失败 +download_sandbox_image_from_modelscope() { + local file="$1" + if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then + echo "sandbox 镜像包 $file 已存在，跳过下载" + return 0 + fi + echo "==> 从 ModelScope 下载 $file ..." + # 优先使用新版 modelscope_hub API（支持 dataset repo） + python3 -c " +from modelscope_hub import HubApi +api = HubApi() +api.download_file( + repo_id='SoraAmami/evalscope-sandbox-images', + repo_type='dataset', + path='$file', + local_dir='$SANDBOX_IMAGES_DIR' +) +" && return 0 + echo "警告：从 ModelScope 下载 $file 失败" + return 1 +} + +load_sandbox_image() { + local file="$1" + local expected_image="$2" + if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$expected_image"; then + echo "镜像 $expected_image 已存在，跳过加载" + return 0 + fi + if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then + echo "==> 加载 $file ..." + docker load -i "$SANDBOX_IMAGES_DIR/$file" + return 0 + fi + return 1 +} + +# 9.1 bigcodebench-sandbox:latest +if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then + if ! load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest"; then + download_sandbox_image_from_modelscope "bigcodebench-sandbox.tar.gz" && \ + load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest" + fi +fi + +# 9.2 python:3.11-slim +if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then + if ! load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim"; then + download_sandbox_image_from_modelscope "python-3.11-slim.tar.gz" && \ + load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim" + fi +fi + +# 9.3 fallback：如果 ModelScope 失败，尝试从 DockerHub / 镜像站拉取 +if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then + echo "==> 尝试 fallback：拉取 bigcodebench 基础镜像并构建 sandbox..." + pull_with_timeout "bigcodebench/bigcodebench-evaluate:latest" 600 || true + if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench/bigcodebench-evaluate:latest"; then + echo "==> 构建 bigcodebench-sandbox..." + mkdir -p /tmp/bigcodebench-sandbox-ctx + docker build -t bigcodebench-sandbox:latest -f - /tmp/bigcodebench-sandbox-ctx <<'EOF' || echo "警告：bigcodebench-sandbox 构建失败" +FROM bigcodebench/bigcodebench-evaluate:latest +ENTRYPOINT [] +CMD ["tail", "-f", "/dev/null"] +EOF + else + echo "警告：bigcodebench 基础镜像不存在，跳过 bigcodebench-sandbox 构建" + fi +fi + +if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then + pull_with_timeout "python:3.11-slim" 300 || true +fi + +# -------------------------------------------------- +# 10. 下载 SWE-Bench 镜像包（可选） +# -------------------------------------------------- +if [ -n "$MODELSCOPE_TOKEN" ]; then + echo "==> 下载 SWE-Bench 镜像包..." + mkdir -p "$SWE_IMAGES_DIR" + modelscope download \ + --repo-type dataset \ + --local_dir "$SWE_IMAGES_DIR" \ + SoraAmami/swe-bench-verified-images || echo "警告：SWE-Bench 镜像下载失败" +fi + +# -------------------------------------------------- +# 11. 加载 SWE-Bench 镜像（如果存在） +# -------------------------------------------------- +if [ -d "$SWE_IMAGES_DIR" ] && ls "$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz 1>/dev/null 2>&1; then + echo "==> 加载 SWE-Bench 镜像..." + TARS=("$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz) + TOTAL=${#TARS[@]} + echo "共找到 $TOTAL 个镜像包" + + IDX=0 + for TAR in "${TARS[@]}"; do + IDX=$((IDX + 1)) + echo "[$IDX/$TOTAL] 加载 $(basename "$TAR")..." + docker load -i "$TAR" + done + + echo "SWE-Bench 镜像加载完成，已加载 $(docker images | grep '^swebench/' | wc -l) 个" +else + echo "跳过 SWE-Bench 镜像加载（未找到镜像包）" +fi + +# -------------------------------------------------- +# 12. 验证部署 +# -------------------------------------------------- +echo "" +echo "========================================" +echo "部署完成！" +echo "========================================" +echo "基础目录: $BASE_DIR" +echo "EvalScope: $EVALSCOPE_DIR" +echo "数据集: $DATASETS_DIR" +echo "Docker 数据: $IMAGES_DIR" +echo "Containerd: $CONTAINERD_DIR" +echo "" +echo "Docker Root Dir:" +docker info 2>/dev/null | grep "Docker Root Dir" || echo "Docker 未运行" +echo "" +echo "已加载镜像:" +docker images | grep -E "evalscope|swebench|bigcodebench|python" || true +echo "" +echo "运行 EvalScope:" +echo " cd $EVALSCOPE_DIR" +echo " docker run -it --rm \\" +echo " --network host \\" +echo " -v $EVALSCOPE_DIR:/opt/evalscope \\" +echo " -v /var/run/docker.sock:/var/run/docker.sock \\" +echo " evalscope-complete-py312:latest \\" +echo " bash" +echo "========================================"