Add sandbox images download from ModelScope; fix Excel sample count total
This commit is contained in:
sora
parent
10eecf787d
commit
991a4a5f7f
170
README.md
170
README.md
@ -76,7 +76,163 @@ python bash/run.py \
|
||||
|
||||
---
|
||||
|
||||
## 3. `bash/run.py` 完整参数
|
||||
## 3. Docker 镜像下载与部署
|
||||
|
||||
根据是否跑 SWE-bench，选择下面两种方式之一。
|
||||
|
||||
### 3.1 方式 A：不测 SWE-bench 的 Docker 配置
|
||||
|
||||
如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像：
|
||||
|
||||
```bash
|
||||
# 1. 暂停服务
|
||||
sudo systemctl stop docker.socket
|
||||
sudo systemctl stop docker
|
||||
|
||||
# 2. 配置 Docker 国内镜像加速
|
||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||
{
|
||||
"registry-mirrors": [
|
||||
"https://docker.m.daocloud.io",
|
||||
"https://docker.1ms.run",
|
||||
"https://hub.rat.dev",
|
||||
"https://docker.1panel.live",
|
||||
"https://dockerproxy.com",
|
||||
"https://hub-mirror.c.163.com",
|
||||
"https://mirror.baidubce.com",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||
"https://docker.nju.edu.cn",
|
||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# 3. 重启 Docker
|
||||
sudo systemctl start docker
|
||||
|
||||
# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖）
|
||||
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||
|
||||
# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须）
|
||||
# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时）
|
||||
python3 -c "
|
||||
from modelscope.hub.file_download import model_file_download
|
||||
model_file_download(
|
||||
model_id='SoraAmami/evalscope-sandbox-images',
|
||||
file_path='bigcodebench-sandbox.tar.gz',
|
||||
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||
)
|
||||
model_file_download(
|
||||
model_id='SoraAmami/evalscope-sandbox-images',
|
||||
file_path='python-3.11-slim.tar.gz',
|
||||
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||
)
|
||||
"
|
||||
docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz
|
||||
docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz
|
||||
|
||||
# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取
|
||||
# docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||
# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||
# FROM bigcodebench/bigcodebench-evaluate:latest
|
||||
# ENTRYPOINT []
|
||||
# CMD ["tail", "-f", "/dev/null"]
|
||||
# EOF
|
||||
# docker pull python:3.11-slim
|
||||
```
|
||||
|
||||
验证：
|
||||
|
||||
```bash
|
||||
docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim'
|
||||
```
|
||||
|
||||
### 3.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径）
|
||||
|
||||
SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘：
|
||||
|
||||
```bash
|
||||
# 1. 暂停服务
|
||||
sudo systemctl stop docker.socket
|
||||
sudo systemctl stop docker
|
||||
sudo systemctl stop containerd
|
||||
|
||||
# 2. 创建新的数据目录
|
||||
mkdir -p /data1/sora/evalscope/docker/images
|
||||
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||
|
||||
# 3. 配置国内镜像加速 + 大容量 data-root
|
||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||
{
|
||||
"data-root": "/data1/sora/evalscope/docker/images",
|
||||
"registry-mirrors": [
|
||||
"https://docker.m.daocloud.io",
|
||||
"https://docker.1ms.run",
|
||||
"https://hub.rat.dev",
|
||||
"https://docker.1panel.live",
|
||||
"https://dockerproxy.com",
|
||||
"https://hub-mirror.c.163.com",
|
||||
"https://mirror.baidubce.com",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||
"https://docker.nju.edu.cn",
|
||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# 4. 配置 containerd 数据目录
|
||||
sudo tee /etc/containerd/config.toml <<-'EOF'
|
||||
root = "/data1/sora/evalscope/docker/containerd"
|
||||
state = "/run/containerd"
|
||||
EOF
|
||||
|
||||
# 5. 重启
|
||||
sudo systemctl reset-failed docker.service
|
||||
sudo systemctl start containerd
|
||||
sudo systemctl start docker
|
||||
|
||||
# 6. 加载 evalscope 环境镜像
|
||||
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||
|
||||
# 7. 准备代码执行 sandbox 镜像
|
||||
# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时）
|
||||
python3 -c "
|
||||
from modelscope.hub.file_download import model_file_download
|
||||
model_file_download(
|
||||
model_id='SoraAmami/evalscope-sandbox-images',
|
||||
file_path='bigcodebench-sandbox.tar.gz',
|
||||
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||
)
|
||||
model_file_download(
|
||||
model_id='SoraAmami/evalscope-sandbox-images',
|
||||
file_path='python-3.11-slim.tar.gz',
|
||||
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||
)
|
||||
"
|
||||
docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz
|
||||
docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz
|
||||
|
||||
# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取
|
||||
# docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||
# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||
# FROM bigcodebench/bigcodebench-evaluate:latest
|
||||
# ENTRYPOINT []
|
||||
# CMD ["tail", "-f", "/dev/null"]
|
||||
# EOF
|
||||
# docker pull python:3.11-slim
|
||||
```
|
||||
|
||||
验证：
|
||||
|
||||
```bash
|
||||
docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. `bash/run.py` 完整参数
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
@ -106,7 +262,7 @@ python bash/run.py \
|
||||
|
||||
---
|
||||
|
||||
## 4. 套件与 Benchmark 覆盖
|
||||
## 5. 套件与 Benchmark 覆盖
|
||||
|
||||
### 4.1 `full` 全量套件
|
||||
|
||||
@ -160,7 +316,7 @@ python bash/run.py \
|
||||
|
||||
---
|
||||
|
||||
## 5. Multi-run 配置
|
||||
## 6. Multi-run 配置
|
||||
|
||||
以下 benchmark 默认会重复跑多次再取平均，次数在 `bash/run.py` 的 `MULTI_RUN_CONFIG` 中定义：
|
||||
|
||||
@ -174,7 +330,7 @@ python bash/run.py \
|
||||
|
||||
---
|
||||
|
||||
## 6. 输出目录与统计
|
||||
## 7. 输出目录与统计
|
||||
|
||||
### 6.1 目录结构
|
||||
|
||||
@ -231,7 +387,7 @@ for f in sorted(glob.glob('output/{folder_name}/*/seed_*/reports/*/*.json')):
|
||||
|
||||
---
|
||||
|
||||
## 7. 快速开始
|
||||
## 8. 快速开始
|
||||
|
||||
### 7.1 安装依赖
|
||||
|
||||
@ -275,7 +431,7 @@ python bash/run.py --help
|
||||
|
||||
---
|
||||
|
||||
## 8. 设计说明
|
||||
## 9. 设计说明
|
||||
|
||||
- **模型无关**：`run.py` 通过 `--model` / `--api-url` 接入任意 OpenAI-compatible 服务。
|
||||
- **配置可覆盖**：YAML 配置 + 命令行参数，未配置的 benchmark 自动使用默认生成参数。
|
||||
@ -285,7 +441,7 @@ python bash/run.py --help
|
||||
|
||||
---
|
||||
|
||||
## 9. 相关文档
|
||||
## 10. 相关文档
|
||||
|
||||
- 详细使用手册：`myread.md`
|
||||
- Docker 构建说明：`DOCKER_BUILD.md`
|
||||
|
||||
1
bash/case/DP4-flash-int8-no-thinking-full.sh
Normal file
1
bash/case/DP4-flash-int8-no-thinking-full.sh
Normal file
@ -0,0 +1 @@
|
||||
python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
|
||||
6
bash/case/DP4-flash-int8-thinking-official.sh
Normal file
6
bash/case/DP4-flash-int8-thinking-official.sh
Normal file
@ -0,0 +1,6 @@
|
||||
python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
|
||||
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking --max-tokens-add 0
|
||||
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking --max-tokens-add 16384
|
||||
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking --max-tokens-add 32768
|
||||
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking --max-tokens-add 65536
|
||||
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking --max-tokens-add 131072
|
||||
@ -348,8 +348,9 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
|
||||
tpot_p99 = percentile(tpots, 99)
|
||||
# The actual sample count we just rebuilt from raw predictions is more
|
||||
# reliable than the (possibly reset) report summary's n_samples.
|
||||
if sample_indexes:
|
||||
n_samples_unique = len(sample_indexes)
|
||||
# For multi-seed / multi-run benchmarks we report the total number of
|
||||
# evaluated predictions (all seeds/runs) rather than unique problem IDs.
|
||||
n_samples_unique = len(latencies)
|
||||
elif summary0:
|
||||
# Fallback to report summary if raw predictions are unavailable
|
||||
latency_mean = summary0.get('latency', {}).get('mean', np.nan)
|
||||
@ -552,11 +553,15 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None,
|
||||
# Add total row
|
||||
total_score = df['得分'].mean()
|
||||
total_time = df['实测时间(h)'].sum()
|
||||
total_samples = df['总样本数'].sum() if '总样本数' in df.columns else np.nan
|
||||
total_tokens = df['累计总tokens'].sum() if '累计总tokens' in df.columns else np.nan
|
||||
total_row = {
|
||||
'分类': '总计',
|
||||
'Benchmark': '',
|
||||
'得分': round(total_score, 4),
|
||||
'实测时间(h)': round(total_time, 4),
|
||||
'总样本数': total_samples if not np.isnan(total_samples) else np.nan,
|
||||
'累计总tokens': total_tokens if not np.isnan(total_tokens) else np.nan,
|
||||
}
|
||||
for col in OUTPUT_COLUMNS:
|
||||
if col not in total_row:
|
||||
|
||||
@ -151,8 +151,12 @@ SUITES = {
|
||||
'official': {
|
||||
'multi': ['aime25', 'aime26', 'live_code_bench'],
|
||||
'single': [
|
||||
'hle', 'mmlu_pro', 'gpqa_diamond', 'longbench_v2',
|
||||
# 'swe_bench_verified',
|
||||
|
||||
'hle',
|
||||
'mmlu_pro',
|
||||
'gpqa_diamond',
|
||||
'longbench_v2',
|
||||
'swe_bench_verified',
|
||||
],
|
||||
'agent': [
|
||||
'tau2_bench'
|
||||
|
||||
659
bash/test.py
659
bash/test.py
@ -1,659 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Unified benchmark runner for EvalScope.
|
||||
|
||||
A single entry point for lite / mid / full / group1 / group2 / group3 evaluations.
|
||||
All tunable parameters can be controlled via command-line arguments.
|
||||
|
||||
Examples:
|
||||
# Full evaluation (all benchmarks, multi-run for stability)
|
||||
python bash/run.py \
|
||||
--model DeepSeek-V4-Flash-Int8 \
|
||||
--api-url http://localhost:30000/v1 \
|
||||
--dataset-dir /data1/sora/evalscope \
|
||||
--output-dir /data1/sora/evalscope/output \
|
||||
--suite full \
|
||||
--limit none
|
||||
|
||||
# Lite smoke test (~5h with full samples)
|
||||
python bash/run.py --suite lite --limit none
|
||||
|
||||
# Run only selected benchmarks
|
||||
python bash/run.py --datasets aime24,gsm8k,arc --limit 20
|
||||
|
||||
# Custom judge model
|
||||
python bash/run.py \
|
||||
--judge-model deepseek-v4-pro \
|
||||
--judge-api-url https://api.deepseek.com/v1 \
|
||||
--judge-api-key sk-xxx
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from evalscope import run_task, TaskConfig
|
||||
from evalscope.api.agent import NativeAgentConfig
|
||||
from evalscope.config import SandboxTaskConfig
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
PROJECT_ROOT = SCRIPT_DIR.parent
|
||||
|
||||
# Make collect_results importable
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
import collect_results as collect_results_module
|
||||
import perf_backup as perf_backup_module
|
||||
|
||||
# ============================================================
|
||||
# Default configuration (override via CLI)
|
||||
# ============================================================
|
||||
|
||||
DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8'
|
||||
DEFAULT_API_URL = 'http://localhost:30000/v1'
|
||||
DEFAULT_DATASET_DIR = str(PROJECT_ROOT)
|
||||
DEFAULT_OUTPUT_DIR = str(PROJECT_ROOT / 'output')
|
||||
DEFAULT_CONFIG = str(PROJECT_ROOT / 'config' / 'dpv4-int8_nothinking.yaml')
|
||||
DEFAULT_TOKENIZER_PATH = '/data1/models/DeepSeek-V4-Flash-INT8'
|
||||
DEFAULT_LIMIT = None
|
||||
DEFAULT_SEED = 42
|
||||
DEFAULT_BATCH_SIZE = 4
|
||||
DEFAULT_ENABLE_THINKING = False
|
||||
|
||||
DEFAULT_JUDGE_MODEL = 'DeepSeek/DeepSeek-V4-Pro'
|
||||
DEFAULT_JUDGE_API_URL = 'https://api.vectron.meta-stone.com/v1'
|
||||
DEFAULT_JUDGE_API_KEY = 'sk-dbd8a665f7634081b87ec409c7636500'
|
||||
DEFAULT_JUDGE_MAX_TOKENS = 10240
|
||||
|
||||
# 长文本 middle-truncation 上限（token 数）。当前默认 128k。
|
||||
DEFAULT_TRUNCATION_TOKENS = 32768 * 4
|
||||
|
||||
# ============================================================
|
||||
# Benchmark suites
|
||||
# ============================================================
|
||||
|
||||
# 多次采样配置：总样本数控制在 ~400-500
|
||||
MULTI_RUN_CONFIG = {
|
||||
'aime24': 12,
|
||||
'aime25': 12,
|
||||
'aime26': 12,
|
||||
'hmmt26': 12,
|
||||
'live_code_bench': 5,
|
||||
'imo_answerbench': 4,
|
||||
'humaneval': 3,
|
||||
'gpqa_diamond': 2,
|
||||
}
|
||||
|
||||
# 能力域完整列表
|
||||
ALL_MULTI_RUN = [
|
||||
# 'humaneval', 'live_code_bench',
|
||||
# 'aime24', 'aime25', 'aime26', 'hmmt26',
|
||||
# 'imo_answerbench', 'gpqa_diamond',
|
||||
]
|
||||
|
||||
|
||||
ALL_SINGLE_RUN = [
|
||||
# 'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa',
|
||||
# 'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro',
|
||||
# 'simple_qa', 'trivia_qa', 'winogrande',
|
||||
# 'openai_mrcr', 'longbench_v2',
|
||||
'swe_bench_verified', 'swe_bench_pro',
|
||||
]
|
||||
ALL_AGENT = [
|
||||
# 'tau2_bench', 'general_fc'
|
||||
]
|
||||
|
||||
# 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡：
|
||||
# Group1: ~61h | Group2: ~62h | Group3: ~55h
|
||||
SUITES = {
|
||||
'full': {
|
||||
'multi': ALL_MULTI_RUN,
|
||||
'single': ALL_SINGLE_RUN,
|
||||
'agent': ALL_AGENT,
|
||||
},
|
||||
'lite': {
|
||||
'multi': ['aime24', 'humaneval'],
|
||||
'single': ['gsm8k', 'arc', 'longbench_v2'],
|
||||
'agent': ['general_fc'],
|
||||
},
|
||||
'mid': {
|
||||
'multi': ['aime24', 'humaneval'],
|
||||
'single': [
|
||||
'live_code_bench', 'bigcodebench', 'competition_math', 'gsm8k',
|
||||
'gpqa_diamond', 'mmlu_pro', 'simple_qa', 'longbench_v2', 'openai_mrcr',
|
||||
],
|
||||
'agent': ['general_fc', 'tau2_bench'],
|
||||
},
|
||||
# 多机组分组，基于 CSV 实测完整时间（已含 multi-run）平衡：
|
||||
# Group1: ~22.7h | Group2: ~25.6h | Group3: ~27.0h | 合计 ~75.2h
|
||||
'group1': {
|
||||
'multi': ['live_code_bench', 'aime24', 'aime25', 'aime26', 'hmmt26', 'imo_answerbench', 'humaneval'],
|
||||
'single': ['bigcodebench', 'competition_math', 'gsm8k', 'drop', 'arc', 'hellaswag', 'winogrande'],
|
||||
'agent': [],
|
||||
},
|
||||
'group2': {
|
||||
'multi': [],
|
||||
'single': ['hle', 'mmlu_pro', 'trivia_qa'],
|
||||
'agent': [],
|
||||
},
|
||||
'group3': {
|
||||
'multi': ['gpqa_diamond'],
|
||||
'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'],
|
||||
'agent': ['tau2_bench', 'general_fc'],
|
||||
},
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# Fixed configuration
|
||||
# ============================================================
|
||||
|
||||
MATH_DATASETS = {
|
||||
'aime24', 'aime25', 'aime26', 'hmmt26',
|
||||
'gsm8k', 'competition_math', 'imo_answerbench',
|
||||
}
|
||||
|
||||
MATH_PROMPT_TEMPLATE = (
|
||||
"{question}\n"
|
||||
"Please reason step by step, and put your final answer within \\boxed{{}}."
|
||||
)
|
||||
|
||||
SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'swe_bench_verified', 'swe_bench_pro'}
|
||||
SANDBOX_CONFIGS = {
|
||||
'bigcodebench': {
|
||||
'image': 'bigcodebench-sandbox:latest',
|
||||
'working_dir': '/tmp',
|
||||
'tools_config': {
|
||||
'shell_executor': {},
|
||||
'python_executor': {}
|
||||
}
|
||||
},
|
||||
'humaneval': {
|
||||
'image': 'python:3.11-slim',
|
||||
'tools_config': {
|
||||
'shell_executor': {},
|
||||
'python_executor': {}
|
||||
}
|
||||
},
|
||||
'swe_bench_verified': {
|
||||
'image': 'swe-bench-sandbox:latest',
|
||||
'working_dir': '/tmp',
|
||||
'tools_config': {
|
||||
'shell_executor': {},
|
||||
'python_executor': {}
|
||||
}
|
||||
},
|
||||
'swe_bench_pro': {
|
||||
'image': 'swe-bench-sandbox:latest',
|
||||
'working_dir': '/tmp',
|
||||
'tools_config': {
|
||||
'shell_executor': {},
|
||||
'python_executor': {}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# CLI parser
|
||||
# ============================================================
|
||||
|
||||
def build_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Unified EvalScope benchmark runner',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog='Suites: full, lite, mid, group1, group2, group3',
|
||||
)
|
||||
|
||||
# Model / API
|
||||
parser.add_argument('--model', default=DEFAULT_MODEL,
|
||||
help='Served model name (default: %(default)s)')
|
||||
parser.add_argument('--api-url', default=DEFAULT_API_URL,
|
||||
help='OpenAI-compatible API URL (default: %(default)s)')
|
||||
|
||||
# Paths
|
||||
parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR,
|
||||
help='Parent directory containing datasets/ subdir (default: %(default)s)')
|
||||
parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR,
|
||||
help='Output root directory (default: %(default)s)')
|
||||
parser.add_argument('--config', default=DEFAULT_CONFIG,
|
||||
help='YAML config path (default: %(default)s)')
|
||||
parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH,
|
||||
help='Local tokenizer path for middle-truncation (default: %(default)s)')
|
||||
|
||||
# Run control
|
||||
parser.add_argument('--suite', default='full', choices=list(SUITES.keys()),
|
||||
help='Benchmark suite to run (default: %(default)s)')
|
||||
parser.add_argument('--datasets', '--benchmarks', dest='datasets', default=None,
|
||||
help='Override suite with comma-separated benchmark names, e.g. aime24,gsm8k')
|
||||
parser.add_argument('--exclude', default=None,
|
||||
help='Comma-separated benchmarks to exclude from the chosen suite')
|
||||
parser.add_argument('--limit', default=None,
|
||||
help='Max samples per benchmark; "none"/"all" for no limit (default: none)')
|
||||
parser.add_argument('--seed', type=int, default=DEFAULT_SEED,
|
||||
help='Random seed (default: %(default)s)')
|
||||
parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE,
|
||||
help='Evaluation batch size (default: %(default)s)')
|
||||
|
||||
# Decoding / thinking
|
||||
parser.add_argument('--thinking', action='store_true', default=None,
|
||||
help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)')
|
||||
parser.add_argument('--no-thinking', dest='thinking', action='store_false',
|
||||
help='Disable thinking mode (default)')
|
||||
|
||||
# Judge model
|
||||
parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
|
||||
help='Judge model name (default: %(default)s)')
|
||||
parser.add_argument('--judge-api-url', default=DEFAULT_JUDGE_API_URL,
|
||||
help='Judge model API URL (default: %(default)s)')
|
||||
parser.add_argument('--judge-api-key', default=DEFAULT_JUDGE_API_KEY,
|
||||
help='Judge model API key')
|
||||
parser.add_argument('--judge-max-tokens', type=int, default=DEFAULT_JUDGE_MAX_TOKENS,
|
||||
help='Judge model max_tokens (default: %(default)s)')
|
||||
|
||||
# Truncation
|
||||
parser.add_argument('--truncation-tokens', type=int, default=DEFAULT_TRUNCATION_TOKENS,
|
||||
help='Middle-truncation token budget for long-context benchmarks (default: %(default)s)')
|
||||
|
||||
# Result collection
|
||||
parser.add_argument('--no-summary', dest='write_summary', action='store_false',
|
||||
help='Skip writing summary Excel/CSV after each benchmark')
|
||||
parser.add_argument('--summary-name', default=None,
|
||||
help='Output summary file name (without extension); defaults to safe model name')
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Middle-truncation helpers
|
||||
# ============================================================
|
||||
|
||||
_TOKENIZER = None
|
||||
|
||||
|
||||
def get_tokenizer(tokenizer_path: str):
|
||||
global _TOKENIZER
|
||||
if _TOKENIZER is None:
|
||||
from transformers import AutoTokenizer
|
||||
try:
|
||||
_TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True)
|
||||
except Exception:
|
||||
_TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True)
|
||||
return _TOKENIZER
|
||||
|
||||
|
||||
def truncate_middle(text: str, max_tokens: int, tokenizer_path: str) -> str:
|
||||
if max_tokens <= 0:
|
||||
return text
|
||||
tokenizer = get_tokenizer(tokenizer_path)
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
if len(token_ids) <= max_tokens:
|
||||
return text
|
||||
keep_head = max_tokens // 2
|
||||
keep_tail = max_tokens - keep_head
|
||||
truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:]
|
||||
return tokenizer.decode(truncated_ids, skip_special_tokens=True)
|
||||
|
||||
|
||||
def _patch_adapters_for_truncation(tokenizer_path: str, truncation_tokens: int):
|
||||
from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter
|
||||
from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter
|
||||
|
||||
_orig_longbench_format = LongBenchV2Adapter.format_prompt_template
|
||||
|
||||
def _patched_longbench_format(self, sample):
|
||||
if sample.metadata and 'context' in sample.metadata:
|
||||
sample.metadata['context'] = truncate_middle(sample.metadata['context'], truncation_tokens, tokenizer_path)
|
||||
return _orig_longbench_format(self, sample)
|
||||
|
||||
LongBenchV2Adapter.format_prompt_template = _patched_longbench_format
|
||||
|
||||
_orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample
|
||||
|
||||
def _patched_mrcr_record(self, record):
|
||||
per_msg_max_tok = 8192
|
||||
if 'prompt' in record:
|
||||
try:
|
||||
prompt_data = json.loads(record['prompt'])
|
||||
if not isinstance(prompt_data, list) or len(prompt_data) == 0:
|
||||
return _orig_mrcr_record(self, record)
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_path)
|
||||
total_tok = sum(
|
||||
len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False))
|
||||
for msg in prompt_data
|
||||
)
|
||||
if total_tok <= truncation_tokens:
|
||||
return _orig_mrcr_record(self, record)
|
||||
|
||||
desired_idx = record.get('desired_msg_index', 0)
|
||||
if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data):
|
||||
desired_idx = 0
|
||||
|
||||
n = len(prompt_data)
|
||||
keep = set()
|
||||
keep.update(range(min(2, n)))
|
||||
keep.update(range(max(0, n - 2), n))
|
||||
window = 2
|
||||
keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1)))
|
||||
keep = sorted(keep)
|
||||
|
||||
new_prompt = []
|
||||
for idx in keep:
|
||||
msg = prompt_data[idx]
|
||||
if isinstance(msg, dict):
|
||||
msg = dict(msg)
|
||||
content = msg.get('content', '')
|
||||
if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok:
|
||||
msg['content'] = truncate_middle(content, per_msg_max_tok, tokenizer_path)
|
||||
new_prompt.append(msg)
|
||||
|
||||
record = dict(record)
|
||||
record['prompt'] = json.dumps(new_prompt)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return _orig_mrcr_record(self, record)
|
||||
|
||||
OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Helpers
|
||||
# ============================================================
|
||||
|
||||
def load_dataset_configs(config_path: str):
|
||||
if not Path(config_path).exists():
|
||||
raise FileNotFoundError(f'Config file not found: {config_path}')
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def configure_thinking(generation_config: dict, enable: bool) -> dict:
|
||||
extra_body = generation_config.get('extra_body', {})
|
||||
chat_template_kwargs = extra_body.get('chat_template_kwargs', {})
|
||||
if enable:
|
||||
chat_template_kwargs['thinking'] = True
|
||||
else:
|
||||
chat_template_kwargs.pop('thinking', None)
|
||||
if chat_template_kwargs:
|
||||
extra_body['chat_template_kwargs'] = chat_template_kwargs
|
||||
if extra_body:
|
||||
generation_config['extra_body'] = extra_body
|
||||
return generation_config
|
||||
|
||||
|
||||
def build_agent_config(agent_cfg: dict) -> NativeAgentConfig:
|
||||
agent_cfg = deepcopy(agent_cfg or {})
|
||||
known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'}
|
||||
kwargs = agent_cfg.pop('kwargs', {})
|
||||
for key in list(agent_cfg.keys()):
|
||||
if key not in known_fields:
|
||||
kwargs[key] = agent_cfg.pop(key)
|
||||
if kwargs:
|
||||
agent_cfg['kwargs'] = kwargs
|
||||
return NativeAgentConfig(**agent_cfg)
|
||||
|
||||
|
||||
def build_task_config(
|
||||
dataset_name: str,
|
||||
ds_cfg: dict,
|
||||
batch_size: int,
|
||||
enable_thinking: bool,
|
||||
seed: int,
|
||||
limit,
|
||||
output_dir: str,
|
||||
model: str,
|
||||
api_url: str,
|
||||
dataset_dir: str,
|
||||
judge_model_args: dict,
|
||||
run_idx: int = 0,
|
||||
) -> TaskConfig:
|
||||
if run_idx > 0:
|
||||
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
|
||||
else:
|
||||
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}'
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
work_dir = str(work_dir)
|
||||
|
||||
generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
|
||||
dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
|
||||
dataset_args.setdefault('shuffle', True)
|
||||
|
||||
if dataset_name in MATH_DATASETS:
|
||||
dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE
|
||||
|
||||
dataset_args_dict = {dataset_name: dataset_args}
|
||||
|
||||
agent_config = None
|
||||
if 'agent_config' in ds_cfg:
|
||||
agent_config = build_agent_config(ds_cfg['agent_config'])
|
||||
|
||||
return TaskConfig(
|
||||
model=model,
|
||||
api_url=api_url,
|
||||
eval_type='openai_api',
|
||||
dataset_dir=dataset_dir,
|
||||
judge_model_args=judge_model_args,
|
||||
seed=seed,
|
||||
limit=limit,
|
||||
collect_perf=True,
|
||||
no_timestamp=True,
|
||||
work_dir=work_dir,
|
||||
use_cache=work_dir,
|
||||
datasets=[dataset_name],
|
||||
generation_config=generation_config,
|
||||
dataset_args=dataset_args_dict,
|
||||
agent_config=agent_config,
|
||||
eval_batch_size=batch_size,
|
||||
sandbox=SandboxTaskConfig(
|
||||
enabled=True,
|
||||
engine='docker',
|
||||
default_config=SANDBOX_CONFIGS.get(dataset_name, {
|
||||
'image': 'python:3.11-slim',
|
||||
'tools_config': {
|
||||
'shell_executor': {},
|
||||
'python_executor': {}
|
||||
}
|
||||
})
|
||||
) if dataset_name in SANDBOX_DATASETS else None,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Main
|
||||
# ============================================================
|
||||
|
||||
def write_summary(output_dir: str, model_name: str, summary_name: str):
|
||||
"""Re-aggregate results across all benchmarks under output_dir."""
|
||||
try:
|
||||
excel_output_dir = PROJECT_ROOT / 'results'
|
||||
collect_results_module.collect_all(
|
||||
Path(output_dir), model_name, summary_name,
|
||||
excel_output_dir=excel_output_dir,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f'WARNING: failed to write summary: {e}')
|
||||
|
||||
|
||||
def backup_after_run(output_dir: str, benchmark: str, model_name: str,
|
||||
work_dir: Path):
|
||||
"""Snapshot the just-finished run's perf summary and predictions."""
|
||||
try:
|
||||
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
|
||||
perf_backup_module.backup_perf_stats(
|
||||
Path(output_dir), benchmark, model_name, report_json,
|
||||
)
|
||||
predictions_dir = work_dir / 'predictions' / model_name
|
||||
perf_backup_module.archive_predictions(
|
||||
Path(output_dir), benchmark, model_name, predictions_dir,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f'WARNING: perf backup for {benchmark} failed: {e}')
|
||||
|
||||
|
||||
def restore_before_run(output_dir: str, benchmark: str, model_name: str,
|
||||
work_dir: Path) -> bool:
|
||||
"""If durable backups exist for a benchmark/model, materialise them into
|
||||
the new ``work_dir`` before evalscope starts so the next run resumes from
|
||||
the larger historical state instead of overwriting it.
|
||||
"""
|
||||
try:
|
||||
predictions_dir = work_dir / 'predictions' / model_name
|
||||
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
|
||||
restored = perf_backup_module.restore_from_backup(
|
||||
Path(output_dir), benchmark, model_name,
|
||||
predictions_dir, report_json,
|
||||
)
|
||||
if restored:
|
||||
print(f'Restored {benchmark}/{model_name} from backup before run')
|
||||
return restored
|
||||
except Exception as e:
|
||||
print(f'WARNING: perf restore for {benchmark} failed: {e}')
|
||||
return False
|
||||
|
||||
|
||||
def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, model_name: str, summary_name: str):
|
||||
"""Run a benchmark task and optionally refresh the summary table.
|
||||
|
||||
After a successful ``run_task()`` we snapshot the cumulative
|
||||
``perf_metrics.summary`` and the per-sample predictions into the durable
|
||||
backup files maintained by ``perf_backup.py`` so checkpoint restarts can
|
||||
recover them.
|
||||
"""
|
||||
dataset_name = task_cfg.datasets[0]
|
||||
work_dir = Path(task_cfg.work_dir)
|
||||
restore_before_run(output_dir, dataset_name, model_name, work_dir)
|
||||
|
||||
start_ts = time.monotonic()
|
||||
try:
|
||||
run_task(task_cfg)
|
||||
finally:
|
||||
elapsed = time.monotonic() - start_ts
|
||||
perf_backup_module.record_active_time(output_dir, dataset_name, model_name, elapsed)
|
||||
print(f'Active time for {dataset_name}: {elapsed:.1f}s (total accumulated)')
|
||||
|
||||
backup_after_run(output_dir, dataset_name, model_name, work_dir)
|
||||
if write_summary_flag:
|
||||
write_summary(output_dir, model_name, summary_name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
# Resolve limit
|
||||
limit = args.limit
|
||||
if limit is not None:
|
||||
if str(limit).lower() in ('none', 'all'):
|
||||
limit = None
|
||||
else:
|
||||
limit = int(limit)
|
||||
|
||||
enable_thinking = DEFAULT_ENABLE_THINKING if args.thinking is None else args.thinking
|
||||
|
||||
# Resolve suite or custom datasets
|
||||
if args.datasets:
|
||||
custom = [d.strip() for d in args.datasets.split(',') if d.strip()]
|
||||
multi_run = [d for d in custom if d in MULTI_RUN_CONFIG]
|
||||
single_run = [d for d in custom if d not in MULTI_RUN_CONFIG]
|
||||
agent = [d for d in custom if d in ALL_AGENT]
|
||||
single_run = [d for d in single_run if d not in ALL_AGENT]
|
||||
else:
|
||||
suite = SUITES[args.suite]
|
||||
multi_run = list(suite['multi'])
|
||||
single_run = list(suite['single'])
|
||||
agent = list(suite['agent'])
|
||||
|
||||
# Apply --exclude
|
||||
if args.exclude:
|
||||
exclude = {d.strip() for d in args.exclude.split(',') if d.strip()}
|
||||
multi_run = [d for d in multi_run if d not in exclude]
|
||||
single_run = [d for d in single_run if d not in exclude]
|
||||
agent = [d for d in agent if d not in exclude]
|
||||
|
||||
judge_model_args = {
|
||||
'model_id': args.judge_model,
|
||||
'api_url': args.judge_api_url,
|
||||
'api_key': args.judge_api_key,
|
||||
'eval_type': 'openai_api',
|
||||
'generation_config': {
|
||||
'temperature': 0.0,
|
||||
'max_tokens': args.judge_max_tokens,
|
||||
},
|
||||
}
|
||||
|
||||
truncation_tokens = args.truncation_tokens
|
||||
_patch_adapters_for_truncation(args.tokenizer_path, truncation_tokens)
|
||||
|
||||
dataset_configs = load_dataset_configs(args.config)
|
||||
|
||||
print('=' * 60)
|
||||
print(f'Config: {args.config}')
|
||||
print(f'Model: {args.model}')
|
||||
print(f'API URL: {args.api_url}')
|
||||
print(f'Dataset Dir: {args.dataset_dir}')
|
||||
print(f'Output Dir: {args.output_dir}')
|
||||
print(f'Suite: {args.suite}')
|
||||
print(f'Limit: {limit if limit is not None else "ALL"}')
|
||||
print(f'Thinking: {enable_thinking}')
|
||||
print(f'Seed: {args.seed}')
|
||||
print(f'Batch Size: {args.batch_size}')
|
||||
print(f'Tokenizer Path: {args.tokenizer_path}')
|
||||
print(f'Truncation Tokens: {truncation_tokens}')
|
||||
print(f'Multi-run datasets: {multi_run}')
|
||||
print(f'Single-run datasets: {single_run}')
|
||||
print(f'Agent datasets: {agent}')
|
||||
print(f'Write summary: {args.write_summary}')
|
||||
print('=' * 60)
|
||||
|
||||
def run_one(dataset_name, run_idx=0):
|
||||
ds_cfg = dataset_configs[dataset_name]
|
||||
task_cfg = build_task_config(
|
||||
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
|
||||
args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
|
||||
run_idx=run_idx,
|
||||
)
|
||||
try:
|
||||
run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, args.summary_name)
|
||||
except Exception as e:
|
||||
print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
|
||||
|
||||
for dataset_name in multi_run:
|
||||
if dataset_name not in dataset_configs:
|
||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
||||
continue
|
||||
num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
|
||||
for run_idx in range(num_runs):
|
||||
print(f"\n{'='*60}")
|
||||
print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
|
||||
print(f"{'='*60}")
|
||||
run_one(dataset_name, run_idx=run_idx)
|
||||
|
||||
for dataset_name in single_run:
|
||||
if dataset_name not in dataset_configs:
|
||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
||||
continue
|
||||
print(f"\n{'='*60}")
|
||||
print(f'Running: {dataset_name} (seed={args.seed})')
|
||||
print(f"{'='*60}")
|
||||
run_one(dataset_name)
|
||||
|
||||
for dataset_name in agent:
|
||||
if dataset_name not in dataset_configs:
|
||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
||||
continue
|
||||
print(f"\n{'='*60}")
|
||||
print(f'Running: {dataset_name} (seed={args.seed})')
|
||||
print(f"{'='*60}")
|
||||
run_one(dataset_name)
|
||||
|
||||
if args.write_summary:
|
||||
write_summary(args.output_dir, args.model, args.summary_name)
|
||||
print('\nAll benchmarks done!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
179
bash/test_e2e.py
179
bash/test_e2e.py
@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""End-to-end offline test for bash/run.py + collect_results + perf_backup.
|
||||
|
||||
This script does NOT call any model API. It uses an existing benchmark
|
||||
directory (aime24/seed_42) and walks through every code path that the real
|
||||
``run.py`` would touch:
|
||||
|
||||
1. backup_perf_stats — confirm a perf summary snapshot is written
|
||||
2. archive_predictions — confirm per-sample records are deduplicated into
|
||||
the archive
|
||||
3. restore_from_backup — confirm a fresh work_dir can be rehydrated from
|
||||
the backup files
|
||||
4. collect_results aggregation — confirm correct metrics are produced
|
||||
5. Whitelist mode — confirm only the requested benchmarks appear in the
|
||||
summary
|
||||
6. hle_low alias — confirm hle_low directory maps to canonical "hle"
|
||||
7. Reset recovery — confirm that even after wiping the predictions file
|
||||
and the report summary, the archive+backup still recover the metrics
|
||||
|
||||
Run from the project root:
|
||||
|
||||
python bash/test_e2e.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
import collect_results as cr # noqa: E402
|
||||
import perf_backup as pb # noqa: E402
|
||||
|
||||
|
||||
PROJECT_ROOT = Path('/data1/sora/evalscope')
|
||||
SOURCE_OUTPUT = PROJECT_ROOT / 'output'
|
||||
|
||||
|
||||
def banner(msg):
|
||||
print('\n' + '=' * 60)
|
||||
print(f' {msg}')
|
||||
print('=' * 60)
|
||||
|
||||
|
||||
def assert_close(actual, expected, name, tol=1e-3):
|
||||
if abs(actual - expected) > tol:
|
||||
raise AssertionError(f'{name}: expected ~{expected}, got {actual}')
|
||||
print(f' OK {name}={actual:.4f}')
|
||||
|
||||
|
||||
def main():
|
||||
banner('Setting up isolated test output dir')
|
||||
test_root = PROJECT_ROOT / 'output_e2e_test'
|
||||
if test_root.exists():
|
||||
shutil.rmtree(test_root)
|
||||
test_root.mkdir(parents=True)
|
||||
|
||||
# Copy the canonical aime24 data into the isolated dir
|
||||
src = SOURCE_OUTPUT / 'aime24' / 'seed_42'
|
||||
dst = test_root / 'aime24' / 'seed_42'
|
||||
shutil.copytree(src, dst)
|
||||
|
||||
# And copy hle_low as hle_low under the test dir (so we exercise the
|
||||
# alias code path).
|
||||
hle_src = SOURCE_OUTPUT / 'hle_low' / 'seed_42'
|
||||
hle_dst = test_root / 'hle_low' / 'seed_42'
|
||||
if hle_src.exists():
|
||||
shutil.copytree(hle_src, hle_dst)
|
||||
|
||||
benchmark = 'aime24'
|
||||
model_name = 'DeepSeek-V4-Flash-Int8'
|
||||
|
||||
banner('1) backup_perf_stats — snapshot summary')
|
||||
report_json = dst / 'reports' / model_name / f'{benchmark}.json'
|
||||
backup_path = pb.backup_perf_stats(test_root, benchmark, model_name, report_json)
|
||||
assert backup_path.exists(), 'backup file was not created'
|
||||
payload = json.loads(backup_path.read_text())
|
||||
assert payload['n_samples'] == 30, f'expected 30 samples, got {payload["n_samples"]}'
|
||||
print(f' OK backup written: {backup_path}, n_samples={payload["n_samples"]}')
|
||||
|
||||
banner('2) archive_predictions — deduplicate into archive')
|
||||
preds = dst / 'predictions' / model_name
|
||||
archive_path = pb.archive_predictions(test_root, benchmark, model_name, preds)
|
||||
assert archive_path.exists(), 'archive file was not created'
|
||||
n_lines = sum(1 for line in archive_path.read_text().splitlines() if line.strip())
|
||||
assert n_lines == 30, f'expected 30 archived samples, got {n_lines}'
|
||||
print(f' OK archive written: {archive_path}, samples={n_lines}')
|
||||
|
||||
banner('3) restore_from_backup — rehydrate a fresh work_dir')
|
||||
fresh_dir = test_root / 'fresh_work' / 'seed_42'
|
||||
fresh_report = fresh_dir / 'reports' / model_name / f'{benchmark}.json'
|
||||
fresh_preds = fresh_dir / 'predictions' / model_name
|
||||
# Pre-create empty structure to mimic what eval_task would do
|
||||
fresh_report.parent.mkdir(parents=True, exist_ok=True)
|
||||
fresh_report.write_text(json.dumps({
|
||||
'name': f'{model_name}@{benchmark}',
|
||||
'perf_metrics': {'summary': {'n_samples': 1}},
|
||||
}))
|
||||
restored = pb.restore_from_backup(
|
||||
test_root, benchmark, model_name, fresh_preds, fresh_report,
|
||||
)
|
||||
assert restored, 'restore_from_backup returned False'
|
||||
restored_report = json.loads(fresh_report.read_text())
|
||||
assert restored_report['perf_metrics']['summary']['n_samples'] == 30, \
|
||||
f'restored n_samples={restored_report["perf_metrics"]["summary"]["n_samples"]}'
|
||||
print(' OK restored report n_samples=30 (matches backup)')
|
||||
|
||||
banner('4) collect_results — aggregate metrics')
|
||||
csv, xlsx = cr.eval_benchmark([benchmark], test_root, model_name)
|
||||
import pandas as pd
|
||||
df = pd.read_csv(csv)
|
||||
aime24 = df[df['Benchmark'] == 'aime24'].iloc[0]
|
||||
# Single seed_42 run, so values match the report directly (no multi-run
|
||||
# averaging). The full-summary CSV averages across run_1..run_11.
|
||||
assert_close(aime24['得分'], 0.6333, 'aime24 score (single seed)')
|
||||
assert_close(aime24['实测时间(h)'], 0.1469, 'aime24 duration (h)')
|
||||
assert_close(aime24['总样本数'], 30, 'aime24 sample count')
|
||||
assert_close(aime24['延迟_mean(s)'], 35.24971, 'aime24 latency mean')
|
||||
assert_close(aime24['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean')
|
||||
assert_close(aime24['TPOT_mean(s)'], 0.02165, 'aime24 TPOT mean')
|
||||
assert_close(aime24['输入tokens_mean'], 119.33, 'aime24 input_tokens_mean')
|
||||
assert_close(aime24['输出tokens_mean'], 1609.8, 'aime24 output_tokens_mean')
|
||||
|
||||
banner('5) Whitelist mode — only the requested benchmarks appear')
|
||||
csv2, _ = cr.eval_benchmark([benchmark], test_root, model_name)
|
||||
df2 = pd.read_csv(csv2)
|
||||
assert df2['Benchmark'].dropna().tolist() == ['aime24'], \
|
||||
f'whitelist did not limit benchmarks: {df2["Benchmark"].tolist()}'
|
||||
print(' OK summary only contains aime24 + total')
|
||||
|
||||
banner('6) hle_low → hle alias')
|
||||
if hle_src.exists():
|
||||
csv3, _ = cr.eval_benchmark(['hle'], test_root, model_name)
|
||||
df3 = pd.read_csv(csv3)
|
||||
assert 'hle' in df3['Benchmark'].tolist(), \
|
||||
f'hle canonical name missing from: {df3["Benchmark"].tolist()}'
|
||||
assert 'hle_low' not in df3['Benchmark'].tolist(), \
|
||||
f'hle_low should be aliased away: {df3["Benchmark"].tolist()}'
|
||||
print(' OK hle_low directory maps to canonical "hle"')
|
||||
|
||||
banner('7) Reset recovery — wipe report+predictions, expect archive to restore')
|
||||
# Wipe the current predictions
|
||||
shutil.rmtree(preds)
|
||||
# Reset report.json
|
||||
data = json.loads(report_json.read_text())
|
||||
data['perf_metrics']['summary']['n_samples'] = 1
|
||||
data['perf_metrics']['summary']['latency']['mean'] = 0.0
|
||||
report_json.write_text(json.dumps(data, indent=2))
|
||||
|
||||
csv4, _ = cr.eval_benchmark([benchmark], test_root, model_name)
|
||||
df4 = pd.read_csv(csv4)
|
||||
aime24_after = df4[df4['Benchmark'] == 'aime24'].iloc[0]
|
||||
assert_close(aime24_after['得分'], 0.6333, 'aime24 score (post-reset)')
|
||||
assert_close(aime24_after['总样本数'], 30, 'aime24 sample count (post-reset)')
|
||||
assert_close(aime24_after['延迟_mean(s)'], 35.24971, 'aime24 latency mean (post-reset)')
|
||||
assert_close(aime24_after['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean (post-reset)')
|
||||
print(' OK archive fully recovered metrics after predictions wipe')
|
||||
|
||||
banner('8) Backup monotonicity — new backup must not overwrite a larger one')
|
||||
# We have a backup with n_samples=30. Now write a fresh report with
|
||||
# n_samples=2 and call backup_perf_stats again — the backup should NOT
|
||||
# be overwritten.
|
||||
data = json.loads(report_json.read_text())
|
||||
data['perf_metrics']['summary']['n_samples'] = 2
|
||||
report_json.write_text(json.dumps(data, indent=2))
|
||||
pb.backup_perf_stats(test_root, benchmark, model_name, report_json)
|
||||
payload2 = json.loads(backup_path.read_text())
|
||||
assert payload2['n_samples'] == 30, \
|
||||
f'backup was clobbered: n_samples now {payload2["n_samples"]}'
|
||||
print(' OK backup preserved n_samples=30 even after a reset run')
|
||||
|
||||
banner('All checks passed')
|
||||
print(f'Test artifacts under {test_root} (kept for inspection)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -101,6 +101,28 @@ def eval_instance(
|
||||
log_dir = Path(log_dir) / 'swebench_log' / instance_id
|
||||
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Resume: skip container evaluation if test output already exists.
|
||||
test_output_path = log_dir / LOG_TEST_OUTPUT
|
||||
if test_output_path.exists():
|
||||
logger.info(f'Found existing test output for {instance_id}, skip container evaluation and reuse log.')
|
||||
try:
|
||||
report = get_eval_report(
|
||||
test_spec=test_spec,
|
||||
prediction=pred,
|
||||
test_log_path=test_output_path,
|
||||
include_tests_status=True,
|
||||
)
|
||||
logger.info(f'report: {report}\n'
|
||||
f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}")
|
||||
return {
|
||||
'completed': True,
|
||||
'resolved': report.get(instance_id, {}).get('resolved', False),
|
||||
'report': report,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to reuse existing test output for {instance_id}: {e}. Will re-evaluate.')
|
||||
|
||||
logger.info(f'Starting evaluation for {instance_id} in log dir {log_dir}...')
|
||||
|
||||
client = DockerClient.from_env()
|
||||
|
||||
110
myread.md
110
myread.md
@ -159,12 +159,65 @@ docker images | grep evalscope-complete-py312
|
||||
| `python:3.11-slim` | 通用代码执行环境 |
|
||||
| `swebench/sweb.eval.x86_64.*` | SWE-bench 每个样本一个实例镜像 |
|
||||
|
||||
#### 2.4.1 配置 Docker（国内镜像加速 + 大容量存储路径）
|
||||
根据是否跑 SWE-bench，选择下面两种 Docker 配置方式之一。
|
||||
|
||||
SWE-bench 镜像数量多、体积大，建议把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘：
|
||||
#### 2.4.1 方式 A：不测 SWE-bench 的 Docker 配置
|
||||
|
||||
如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像：
|
||||
|
||||
```bash
|
||||
# 1. 停止 Docker
|
||||
# 1. 暂停服务
|
||||
sudo systemctl stop docker.socket
|
||||
sudo systemctl stop docker
|
||||
|
||||
# 2. 配置 Docker 国内镜像加速
|
||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||
{
|
||||
"registry-mirrors": [
|
||||
"https://docker.m.daocloud.io",
|
||||
"https://docker.1ms.run",
|
||||
"https://hub.rat.dev",
|
||||
"https://docker.1panel.live",
|
||||
"https://dockerproxy.com",
|
||||
"https://hub-mirror.c.163.com",
|
||||
"https://mirror.baidubce.com",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||
"https://docker.nju.edu.cn",
|
||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# 3. 重启 Docker
|
||||
sudo systemctl start docker
|
||||
|
||||
# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖）
|
||||
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||
|
||||
# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须）
|
||||
docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||
docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||
ENTRYPOINT []
|
||||
CMD ["tail", "-f", "/dev/null"]
|
||||
EOF
|
||||
|
||||
docker pull python:3.11-slim
|
||||
```
|
||||
|
||||
验证：
|
||||
|
||||
```bash
|
||||
docker images | grep -E 'evalscope-complete-py312|bigcodebench|python:3.11-slim'
|
||||
```
|
||||
|
||||
#### 2.4.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径）
|
||||
|
||||
SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘，否则默认盘很快会被占满：
|
||||
|
||||
```bash
|
||||
# 1. 暂停服务
|
||||
sudo systemctl stop docker.socket
|
||||
sudo systemctl stop docker
|
||||
sudo systemctl stop containerd
|
||||
@ -173,7 +226,7 @@ sudo systemctl stop containerd
|
||||
mkdir -p /data1/sora/evalscope/docker/images
|
||||
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||
|
||||
# 3. 配置国内镜像加速（多填几个，自动轮询）
|
||||
# 3. 配置国内镜像加速 + 大容量 data-root
|
||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||
{
|
||||
"data-root": "/data1/sora/evalscope/docker/images",
|
||||
@ -203,22 +256,18 @@ EOF
|
||||
sudo systemctl reset-failed docker.service
|
||||
sudo systemctl start containerd
|
||||
sudo systemctl start docker
|
||||
```
|
||||
|
||||
#### 2.4.2 拉取/构建代码执行镜像
|
||||
# 6. 加载 evalscope 环境镜像
|
||||
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||
|
||||
```bash
|
||||
# bigcodebench 官方镜像
|
||||
# 7. 准备代码执行 sandbox 镜像
|
||||
docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||
|
||||
# 构建守护态 sandbox
|
||||
docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||
ENTRYPOINT []
|
||||
CMD ["tail", "-f", "/dev/null"]
|
||||
EOF
|
||||
|
||||
# 通用 Python 执行环境
|
||||
docker pull python:3.11-slim
|
||||
```
|
||||
|
||||
@ -267,25 +316,42 @@ python bash/pull_swe_bench_images.py \
|
||||
> 2. 降低 `--max-workers` 到 1；
|
||||
> 3. 使用 `tmux` 挂后台运行，避免 SSH 断连导致中断。
|
||||
|
||||
#### 2.4.4 SWE-bench 镜像的本地备份与迁移（可选）
|
||||
#### 2.4.4 SWE-bench 镜像的备份与迁移（可选）
|
||||
|
||||
SWE-bench 全量镜像约 500 个、总大小约 **2TB**，无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器，建议用移动硬盘或内网直接拷贝 Docker `data-root`，或使用下面脚本分批导出/导入：
|
||||
SWE-bench Verified 全量约 500 个镜像、本地实际占用约 **160GB**。迁移到另一台机器有三种方式：
|
||||
|
||||
**源机器：打包**
|
||||
**方式一：从 ModelScope 下载（推荐，已上传）**
|
||||
|
||||
`SoraAmami/swe-bench-verified-images` 已包含分批导出的 `docker save` tar.gz，目标机器直接下载加载即可：
|
||||
|
||||
```bash
|
||||
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
||||
```
|
||||
# 下载
|
||||
modelscope download \
|
||||
--repo-type dataset \
|
||||
--local_dir /data1/sora/evalscope/docker/swe_images \
|
||||
SoraAmami/swe-bench-verified-images
|
||||
|
||||
脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。
|
||||
|
||||
**目标机器：加载**
|
||||
|
||||
```bash
|
||||
# 加载
|
||||
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||
```
|
||||
|
||||
> 注意：分批 tar 会占用额外磁盘空间（约 2TB），请确保目标盘容量足够。
|
||||
**方式二：本地分批导出/导入**
|
||||
|
||||
如果已有本地镜像，可用脚本打包后复制到目标机器：
|
||||
|
||||
```bash
|
||||
# 源机器打包（每 50 个一批）
|
||||
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
||||
|
||||
# 目标机器加载
|
||||
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||
```
|
||||
|
||||
**方式三：直接拷贝 Docker data-root**
|
||||
|
||||
适用于内网或移动硬盘，把 `/data1/sora/evalscope/docker/images` 和 `/data1/sora/evalscope/docker/containerd` 完整复制到目标机器相同路径，再启动 Docker。注意这种方式对 Docker 版本和路径要求严格，不如前两种稳定。
|
||||
|
||||
> 注意：分批 tar 会临时占用额外磁盘空间，加载完成后可删除 tar 包。
|
||||
|
||||
---
|
||||
|
||||
|
||||
58
scripts/Dockerfile.bigcodebench-sandbox
Normal file
58
scripts/Dockerfile.bigcodebench-sandbox
Normal file
@ -0,0 +1,58 @@
|
||||
# bigcodebench-sandbox
|
||||
# Minimal sandbox for running BigCodeBench/HumanEval code inside evalscope.
|
||||
# Based on https://github.com/bigcode-project/bigcodebench/blob/main/Docker/Evaluate.Dockerfile
|
||||
# but stripped to only the runtime dependencies needed to execute benchmark tests.
|
||||
# Uses python:3.9-slim because some pinned packages (e.g. numba==0.55.0) do not have
|
||||
# wheels for newer Python versions.
|
||||
# Uses Chinese mirrors for apt/pip to speed up builds inside mainland network.
|
||||
|
||||
FROM python:3.9-slim
|
||||
|
||||
# Use Chinese Debian mirror
|
||||
RUN sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list.d/debian.sources && \
|
||||
sed -i 's|http://deb.debian.org/debian-security|http://mirrors.tuna.tsinghua.edu.cn/debian-security|g' /etc/apt/sources.list.d/debian.sources && \
|
||||
sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list || true
|
||||
|
||||
# Configure Chinese PyPI mirror
|
||||
RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
|
||||
pip config set global.timeout 2000
|
||||
|
||||
# Install system dependencies needed by the scientific/Python packages
|
||||
RUN apt-get update && apt-get install -y \
|
||||
git \
|
||||
g++ \
|
||||
python3-tk \
|
||||
zip \
|
||||
unzip \
|
||||
procps \
|
||||
r-base \
|
||||
libgdal-dev \
|
||||
libfreetype6-dev \
|
||||
libpng-dev \
|
||||
pkg-config \
|
||||
python3-dev \
|
||||
python3-matplotlib \
|
||||
libgl1 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Upgrade pip
|
||||
RUN pip install --upgrade pip
|
||||
|
||||
# Add a non-root user (matches upstream image conventions)
|
||||
RUN adduser --disabled-password --gecos "" bigcodebenchuser
|
||||
|
||||
# Copy and install the BigCodeBench evaluation requirements.
|
||||
# These are the 70+ Python libraries referenced by the benchmark test cases.
|
||||
COPY requirements-eval.txt /tmp/requirements-eval.txt
|
||||
RUN pip install -I --timeout 2000 -r /tmp/requirements-eval.txt
|
||||
|
||||
# Ensure a compatible datasets version
|
||||
RUN pip install datasets==2.17.0
|
||||
|
||||
WORKDIR /app
|
||||
RUN chown -R bigcodebenchuser:bigcodebenchuser /app && \
|
||||
chmod -R 777 /app
|
||||
|
||||
# For evalscope sandbox: keep container alive
|
||||
ENTRYPOINT []
|
||||
CMD ["tail", "-f", "/dev/null"]
|
||||
292
scripts/deploy_evalscope.sh
Executable file
292
scripts/deploy_evalscope.sh
Executable file
@ -0,0 +1,292 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# EvalScope 完整部署脚本
|
||||
# 用法: bash deploy_evalscope.sh <BASE_DIR> [TOKEN]
|
||||
# 示例: bash deploy_evalscope.sh /data1/sora ms-3d554a39-6e07-496d-8022-0b0ee64a6389
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# --------------------------------------------------
|
||||
# 1. 参数解析
|
||||
# --------------------------------------------------
|
||||
BASE_DIR="${1:-/data1/sora}" # 基础目录，默认 /data1/sora
|
||||
DEFAULT_TOKEN="ms-3d554a39-6e07-496d-8022-0b0ee64a6389"
|
||||
MODELSCOPE_TOKEN="${2:-$DEFAULT_TOKEN}"
|
||||
|
||||
# 派生路径
|
||||
EVALSCOPE_DIR="$BASE_DIR/evalscope"
|
||||
DOCKER_DIR="$EVALSCOPE_DIR/docker"
|
||||
IMAGES_DIR="$DOCKER_DIR/images"
|
||||
CONTAINERD_DIR="$DOCKER_DIR/containerd"
|
||||
SWE_IMAGES_DIR="$DOCKER_DIR/swe_images"
|
||||
DATASETS_DIR="$EVALSCOPE_DIR/datasets"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 2. 安装依赖
|
||||
# --------------------------------------------------
|
||||
echo "==> 安装 modelscope..."
|
||||
pip install modelscope
|
||||
|
||||
# --------------------------------------------------
|
||||
# 3. 登录 ModelScope
|
||||
# --------------------------------------------------
|
||||
if [ -n "$MODELSCOPE_TOKEN" ]; then
|
||||
echo "==> 登录 ModelScope..."
|
||||
modelscope login --token "$MODELSCOPE_TOKEN"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# 4. 下载数据集
|
||||
# --------------------------------------------------
|
||||
echo "==> 下载 evalscope 数据集..."
|
||||
mkdir -p "$DATASETS_DIR"
|
||||
python3 -c "
|
||||
from modelscope.hub.snapshot_download import snapshot_download
|
||||
snapshot_download(
|
||||
'SoraAmami/evalscope-datasets',
|
||||
repo_type='dataset',
|
||||
cache_dir='$EVALSCOPE_DIR',
|
||||
local_dir='$DATASETS_DIR'
|
||||
)
|
||||
"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 5. 下载 Docker 镜像包
|
||||
# --------------------------------------------------
|
||||
echo "==> 下载 evalscope Docker 镜像..."
|
||||
mkdir -p "$DOCKER_DIR"
|
||||
python3 -c "
|
||||
from modelscope.hub.file_download import model_file_download
|
||||
model_file_download(
|
||||
model_id='SoraAmami/evalscope-docker',
|
||||
file_path='evalscope-complete-py312.tar.gz',
|
||||
local_dir='$DOCKER_DIR'
|
||||
)
|
||||
"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 6. 配置 Docker 和 containerd 数据目录
|
||||
# --------------------------------------------------
|
||||
echo "==> 配置 Docker 和 containerd..."
|
||||
|
||||
# 停止服务
|
||||
sudo systemctl stop docker.socket 2>/dev/null || true
|
||||
sudo systemctl stop docker 2>/dev/null || true
|
||||
sudo systemctl stop containerd 2>/dev/null || true
|
||||
|
||||
# 创建数据目录
|
||||
mkdir -p "$IMAGES_DIR"
|
||||
mkdir -p "$CONTAINERD_DIR"
|
||||
|
||||
# 配置 Docker
|
||||
echo "==> 写入 Docker 配置..."
|
||||
sudo tee /etc/docker/daemon.json <<EOF
|
||||
{
|
||||
"data-root": "$IMAGES_DIR",
|
||||
"features": {
|
||||
"containerd-snapshotter": true
|
||||
},
|
||||
"registry-mirrors": [
|
||||
"https://docker.m.daocloud.io",
|
||||
"https://docker.1ms.run",
|
||||
"https://hub.rat.dev",
|
||||
"https://docker.1panel.live",
|
||||
"https://dockerproxy.com",
|
||||
"https://hub-mirror.c.163.com",
|
||||
"https://mirror.baidubce.com",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||
"https://docker.nju.edu.cn",
|
||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# 配置 containerd
|
||||
echo "==> 写入 containerd 配置..."
|
||||
sudo tee /etc/containerd/config.toml <<EOF
|
||||
root = "$CONTAINERD_DIR"
|
||||
state = "/run/containerd"
|
||||
EOF
|
||||
|
||||
# 重启服务
|
||||
sudo systemctl reset-failed docker.service 2>/dev/null || true
|
||||
sudo systemctl start containerd
|
||||
sudo systemctl start docker
|
||||
|
||||
# --------------------------------------------------
|
||||
# 7. 加载 evalscope 环境镜像
|
||||
# --------------------------------------------------
|
||||
echo "==> 加载 evalscope Docker 镜像..."
|
||||
docker load -i "$DOCKER_DIR/evalscope-complete-py312.tar.gz"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 8. 克隆代码仓库
|
||||
# --------------------------------------------------
|
||||
echo "==> 克隆 evalscope 代码..."
|
||||
cd "$EVALSCOPE_DIR"
|
||||
if [ ! -d "evalstone" ]; then
|
||||
git clone https://git.meta-stone.net/sora/evalstone.git
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# 9. 准备 sandbox 镜像
|
||||
# --------------------------------------------------
|
||||
SANDBOX_IMAGES_DIR="$DOCKER_DIR/sandbox_images"
|
||||
mkdir -p "$SANDBOX_IMAGES_DIR"
|
||||
|
||||
# 函数：带超时拉取镜像，本地已有则跳过
|
||||
pull_with_timeout() {
|
||||
local image="$1"
|
||||
local timeout_sec="${2:-600}"
|
||||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$image"; then
|
||||
echo "镜像 $image 已存在，跳过 pull"
|
||||
return 0
|
||||
fi
|
||||
echo "拉取 $image (超时 ${timeout_sec}s)..."
|
||||
if timeout "$timeout_sec" docker pull "$image"; then
|
||||
echo "$image 拉取成功"
|
||||
return 0
|
||||
else
|
||||
echo "警告：$image 拉取失败或超时"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 优先从 ModelScope 下载预打包的 sandbox 镜像，避免 DockerHub 拉取慢/失败
|
||||
download_sandbox_image_from_modelscope() {
|
||||
local file="$1"
|
||||
if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then
|
||||
echo "sandbox 镜像包 $file 已存在，跳过下载"
|
||||
return 0
|
||||
fi
|
||||
echo "==> 从 ModelScope 下载 $file ..."
|
||||
# 优先使用新版 modelscope_hub API（支持 dataset repo）
|
||||
python3 -c "
|
||||
from modelscope_hub import HubApi
|
||||
api = HubApi()
|
||||
api.download_file(
|
||||
repo_id='SoraAmami/evalscope-sandbox-images',
|
||||
repo_type='dataset',
|
||||
path='$file',
|
||||
local_dir='$SANDBOX_IMAGES_DIR'
|
||||
)
|
||||
" && return 0
|
||||
echo "警告：从 ModelScope 下载 $file 失败"
|
||||
return 1
|
||||
}
|
||||
|
||||
load_sandbox_image() {
|
||||
local file="$1"
|
||||
local expected_image="$2"
|
||||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$expected_image"; then
|
||||
echo "镜像 $expected_image 已存在，跳过加载"
|
||||
return 0
|
||||
fi
|
||||
if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then
|
||||
echo "==> 加载 $file ..."
|
||||
docker load -i "$SANDBOX_IMAGES_DIR/$file"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# 9.1 bigcodebench-sandbox:latest
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then
|
||||
if ! load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest"; then
|
||||
download_sandbox_image_from_modelscope "bigcodebench-sandbox.tar.gz" && \
|
||||
load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 9.2 python:3.11-slim
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then
|
||||
if ! load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim"; then
|
||||
download_sandbox_image_from_modelscope "python-3.11-slim.tar.gz" && \
|
||||
load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 9.3 fallback：如果 ModelScope 失败，尝试从 DockerHub / 镜像站拉取
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then
|
||||
echo "==> 尝试 fallback：拉取 bigcodebench 基础镜像并构建 sandbox..."
|
||||
pull_with_timeout "bigcodebench/bigcodebench-evaluate:latest" 600 || true
|
||||
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench/bigcodebench-evaluate:latest"; then
|
||||
echo "==> 构建 bigcodebench-sandbox..."
|
||||
mkdir -p /tmp/bigcodebench-sandbox-ctx
|
||||
docker build -t bigcodebench-sandbox:latest -f - /tmp/bigcodebench-sandbox-ctx <<'EOF' || echo "警告：bigcodebench-sandbox 构建失败"
|
||||
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||
ENTRYPOINT []
|
||||
CMD ["tail", "-f", "/dev/null"]
|
||||
EOF
|
||||
else
|
||||
echo "警告：bigcodebench 基础镜像不存在，跳过 bigcodebench-sandbox 构建"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then
|
||||
pull_with_timeout "python:3.11-slim" 300 || true
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# 10. 下载 SWE-Bench 镜像包（可选）
|
||||
# --------------------------------------------------
|
||||
if [ -n "$MODELSCOPE_TOKEN" ]; then
|
||||
echo "==> 下载 SWE-Bench 镜像包..."
|
||||
mkdir -p "$SWE_IMAGES_DIR"
|
||||
modelscope download \
|
||||
--repo-type dataset \
|
||||
--local_dir "$SWE_IMAGES_DIR" \
|
||||
SoraAmami/swe-bench-verified-images || echo "警告：SWE-Bench 镜像下载失败"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# 11. 加载 SWE-Bench 镜像（如果存在）
|
||||
# --------------------------------------------------
|
||||
if [ -d "$SWE_IMAGES_DIR" ] && ls "$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz 1>/dev/null 2>&1; then
|
||||
echo "==> 加载 SWE-Bench 镜像..."
|
||||
TARS=("$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz)
|
||||
TOTAL=${#TARS[@]}
|
||||
echo "共找到 $TOTAL 个镜像包"
|
||||
|
||||
IDX=0
|
||||
for TAR in "${TARS[@]}"; do
|
||||
IDX=$((IDX + 1))
|
||||
echo "[$IDX/$TOTAL] 加载 $(basename "$TAR")..."
|
||||
docker load -i "$TAR"
|
||||
done
|
||||
|
||||
echo "SWE-Bench 镜像加载完成，已加载 $(docker images | grep '^swebench/' | wc -l) 个"
|
||||
else
|
||||
echo "跳过 SWE-Bench 镜像加载（未找到镜像包）"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# 12. 验证部署
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "部署完成！"
|
||||
echo "========================================"
|
||||
echo "基础目录: $BASE_DIR"
|
||||
echo "EvalScope: $EVALSCOPE_DIR"
|
||||
echo "数据集: $DATASETS_DIR"
|
||||
echo "Docker 数据: $IMAGES_DIR"
|
||||
echo "Containerd: $CONTAINERD_DIR"
|
||||
echo ""
|
||||
echo "Docker Root Dir:"
|
||||
docker info 2>/dev/null | grep "Docker Root Dir" || echo "Docker 未运行"
|
||||
echo ""
|
||||
echo "已加载镜像:"
|
||||
docker images | grep -E "evalscope|swebench|bigcodebench|python" || true
|
||||
echo ""
|
||||
echo "运行 EvalScope:"
|
||||
echo " cd $EVALSCOPE_DIR"
|
||||
echo " docker run -it --rm \\"
|
||||
echo " --network host \\"
|
||||
echo " -v $EVALSCOPE_DIR:/opt/evalscope \\"
|
||||
echo " -v /var/run/docker.sock:/var/run/docker.sock \\"
|
||||
echo " evalscope-complete-py312:latest \\"
|
||||
echo " bash"
|
||||
echo "========================================"
|
||||
Loading…
x
Reference in New Issue
Block a user