Add sandbox images download from ModelScope; fix Excel sample count total
This commit is contained in:
sora
parent
10eecf787d
commit
991a4a5f7f
170
README.md
170
README.md
@ -76,7 +76,163 @@ python bash/run.py \
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. `bash/run.py` 完整参数
|
## 3. Docker 镜像下载与部署
|
||||||
|
|
||||||
|
根据是否跑 SWE-bench，选择下面两种方式之一。
|
||||||
|
|
||||||
|
### 3.1 方式 A：不测 SWE-bench 的 Docker 配置
|
||||||
|
|
||||||
|
如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. 暂停服务
|
||||||
|
sudo systemctl stop docker.socket
|
||||||
|
sudo systemctl stop docker
|
||||||
|
|
||||||
|
# 2. 配置 Docker 国内镜像加速
|
||||||
|
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||||
|
{
|
||||||
|
"registry-mirrors": [
|
||||||
|
"https://docker.m.daocloud.io",
|
||||||
|
"https://docker.1ms.run",
|
||||||
|
"https://hub.rat.dev",
|
||||||
|
"https://docker.1panel.live",
|
||||||
|
"https://dockerproxy.com",
|
||||||
|
"https://hub-mirror.c.163.com",
|
||||||
|
"https://mirror.baidubce.com",
|
||||||
|
"https://docker.mirrors.ustc.edu.cn",
|
||||||
|
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||||
|
"https://docker.nju.edu.cn",
|
||||||
|
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 3. 重启 Docker
|
||||||
|
sudo systemctl start docker
|
||||||
|
|
||||||
|
# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖）
|
||||||
|
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||||
|
|
||||||
|
# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须）
|
||||||
|
# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时）
|
||||||
|
python3 -c "
|
||||||
|
from modelscope.hub.file_download import model_file_download
|
||||||
|
model_file_download(
|
||||||
|
model_id='SoraAmami/evalscope-sandbox-images',
|
||||||
|
file_path='bigcodebench-sandbox.tar.gz',
|
||||||
|
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||||
|
)
|
||||||
|
model_file_download(
|
||||||
|
model_id='SoraAmami/evalscope-sandbox-images',
|
||||||
|
file_path='python-3.11-slim.tar.gz',
|
||||||
|
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||||
|
)
|
||||||
|
"
|
||||||
|
docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz
|
||||||
|
docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz
|
||||||
|
|
||||||
|
# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取
|
||||||
|
# docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||||
|
# FROM bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
# ENTRYPOINT []
|
||||||
|
# CMD ["tail", "-f", "/dev/null"]
|
||||||
|
# EOF
|
||||||
|
# docker pull python:3.11-slim
|
||||||
|
```
|
||||||
|
|
||||||
|
验证：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim'
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径）
|
||||||
|
|
||||||
|
SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. 暂停服务
|
||||||
|
sudo systemctl stop docker.socket
|
||||||
|
sudo systemctl stop docker
|
||||||
|
sudo systemctl stop containerd
|
||||||
|
|
||||||
|
# 2. 创建新的数据目录
|
||||||
|
mkdir -p /data1/sora/evalscope/docker/images
|
||||||
|
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||||
|
|
||||||
|
# 3. 配置国内镜像加速 + 大容量 data-root
|
||||||
|
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||||
|
{
|
||||||
|
"data-root": "/data1/sora/evalscope/docker/images",
|
||||||
|
"registry-mirrors": [
|
||||||
|
"https://docker.m.daocloud.io",
|
||||||
|
"https://docker.1ms.run",
|
||||||
|
"https://hub.rat.dev",
|
||||||
|
"https://docker.1panel.live",
|
||||||
|
"https://dockerproxy.com",
|
||||||
|
"https://hub-mirror.c.163.com",
|
||||||
|
"https://mirror.baidubce.com",
|
||||||
|
"https://docker.mirrors.ustc.edu.cn",
|
||||||
|
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||||
|
"https://docker.nju.edu.cn",
|
||||||
|
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 4. 配置 containerd 数据目录
|
||||||
|
sudo tee /etc/containerd/config.toml <<-'EOF'
|
||||||
|
root = "/data1/sora/evalscope/docker/containerd"
|
||||||
|
state = "/run/containerd"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 5. 重启
|
||||||
|
sudo systemctl reset-failed docker.service
|
||||||
|
sudo systemctl start containerd
|
||||||
|
sudo systemctl start docker
|
||||||
|
|
||||||
|
# 6. 加载 evalscope 环境镜像
|
||||||
|
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||||
|
|
||||||
|
# 7. 准备代码执行 sandbox 镜像
|
||||||
|
# 推荐：从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时）
|
||||||
|
python3 -c "
|
||||||
|
from modelscope.hub.file_download import model_file_download
|
||||||
|
model_file_download(
|
||||||
|
model_id='SoraAmami/evalscope-sandbox-images',
|
||||||
|
file_path='bigcodebench-sandbox.tar.gz',
|
||||||
|
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||||
|
)
|
||||||
|
model_file_download(
|
||||||
|
model_id='SoraAmami/evalscope-sandbox-images',
|
||||||
|
file_path='python-3.11-slim.tar.gz',
|
||||||
|
local_dir='/data1/sora/evalscope/docker/sandbox_images'
|
||||||
|
)
|
||||||
|
"
|
||||||
|
docker load -i /data1/sora/evalscope/docker/sandbox_images/bigcodebench-sandbox.tar.gz
|
||||||
|
docker load -i /data1/sora/evalscope/docker/sandbox_images/python-3.11-slim.tar.gz
|
||||||
|
|
||||||
|
# 备选：如果 ModelScope 下载失败，再尝试从 DockerHub / 镜像站拉取
|
||||||
|
# docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
# docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||||
|
# FROM bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
# ENTRYPOINT []
|
||||||
|
# CMD ["tail", "-f", "/dev/null"]
|
||||||
|
# EOF
|
||||||
|
# docker pull python:3.11-slim
|
||||||
|
```
|
||||||
|
|
||||||
|
验证：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.11-slim'
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. `bash/run.py` 完整参数
|
||||||
|
|
||||||
| 参数 | 默认值 | 说明 |
|
| 参数 | 默认值 | 说明 |
|
||||||
|------|--------|------|
|
|------|--------|------|
|
||||||
@ -106,7 +262,7 @@ python bash/run.py \
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 4. 套件与 Benchmark 覆盖
|
## 5. 套件与 Benchmark 覆盖
|
||||||
|
|
||||||
### 4.1 `full` 全量套件
|
### 4.1 `full` 全量套件
|
||||||
|
|
||||||
@ -160,7 +316,7 @@ python bash/run.py \
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 5. Multi-run 配置
|
## 6. Multi-run 配置
|
||||||
|
|
||||||
以下 benchmark 默认会重复跑多次再取平均，次数在 `bash/run.py` 的 `MULTI_RUN_CONFIG` 中定义：
|
以下 benchmark 默认会重复跑多次再取平均，次数在 `bash/run.py` 的 `MULTI_RUN_CONFIG` 中定义：
|
||||||
|
|
||||||
@ -174,7 +330,7 @@ python bash/run.py \
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 6. 输出目录与统计
|
## 7. 输出目录与统计
|
||||||
|
|
||||||
### 6.1 目录结构
|
### 6.1 目录结构
|
||||||
|
|
||||||
@ -231,7 +387,7 @@ for f in sorted(glob.glob('output/{folder_name}/*/seed_*/reports/*/*.json')):
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 7. 快速开始
|
## 8. 快速开始
|
||||||
|
|
||||||
### 7.1 安装依赖
|
### 7.1 安装依赖
|
||||||
|
|
||||||
@ -275,7 +431,7 @@ python bash/run.py --help
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 8. 设计说明
|
## 9. 设计说明
|
||||||
|
|
||||||
- **模型无关**：`run.py` 通过 `--model` / `--api-url` 接入任意 OpenAI-compatible 服务。
|
- **模型无关**：`run.py` 通过 `--model` / `--api-url` 接入任意 OpenAI-compatible 服务。
|
||||||
- **配置可覆盖**：YAML 配置 + 命令行参数，未配置的 benchmark 自动使用默认生成参数。
|
- **配置可覆盖**：YAML 配置 + 命令行参数，未配置的 benchmark 自动使用默认生成参数。
|
||||||
@ -285,7 +441,7 @@ python bash/run.py --help
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 9. 相关文档
|
## 10. 相关文档
|
||||||
|
|
||||||
- 详细使用手册：`myread.md`
|
- 详细使用手册：`myread.md`
|
||||||
- Docker 构建说明：`DOCKER_BUILD.md`
|
- Docker 构建说明：`DOCKER_BUILD.md`
|
||||||
|
|||||||
1
bash/case/DP4-flash-int8-no-thinking-full.sh
Normal file
1
bash/case/DP4-flash-int8-no-thinking-full.sh
Normal file
@ -0,0 +1 @@
|
|||||||
|
python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
|
||||||
6
bash/case/DP4-flash-int8-thinking-official.sh
Normal file
6
bash/case/DP4-flash-int8-thinking-official.sh
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
python bash/run.py --datasets swe_bench_verified --folder-name DP4-flash-int8-not-thinking
|
||||||
|
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add0 --thinking --max-tokens-add 0
|
||||||
|
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add16k --thinking --max-tokens-add 16384
|
||||||
|
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add32k --thinking --max-tokens-add 32768
|
||||||
|
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add64k --thinking --max-tokens-add 65536
|
||||||
|
python bash/run.py --suite official --folder-name DP4-flash-int8-thinking-add128k --thinking --max-tokens-add 131072
|
||||||
@ -348,8 +348,9 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
|
|||||||
tpot_p99 = percentile(tpots, 99)
|
tpot_p99 = percentile(tpots, 99)
|
||||||
# The actual sample count we just rebuilt from raw predictions is more
|
# The actual sample count we just rebuilt from raw predictions is more
|
||||||
# reliable than the (possibly reset) report summary's n_samples.
|
# reliable than the (possibly reset) report summary's n_samples.
|
||||||
if sample_indexes:
|
# For multi-seed / multi-run benchmarks we report the total number of
|
||||||
n_samples_unique = len(sample_indexes)
|
# evaluated predictions (all seeds/runs) rather than unique problem IDs.
|
||||||
|
n_samples_unique = len(latencies)
|
||||||
elif summary0:
|
elif summary0:
|
||||||
# Fallback to report summary if raw predictions are unavailable
|
# Fallback to report summary if raw predictions are unavailable
|
||||||
latency_mean = summary0.get('latency', {}).get('mean', np.nan)
|
latency_mean = summary0.get('latency', {}).get('mean', np.nan)
|
||||||
@ -552,11 +553,15 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None,
|
|||||||
# Add total row
|
# Add total row
|
||||||
total_score = df['得分'].mean()
|
total_score = df['得分'].mean()
|
||||||
total_time = df['实测时间(h)'].sum()
|
total_time = df['实测时间(h)'].sum()
|
||||||
|
total_samples = df['总样本数'].sum() if '总样本数' in df.columns else np.nan
|
||||||
|
total_tokens = df['累计总tokens'].sum() if '累计总tokens' in df.columns else np.nan
|
||||||
total_row = {
|
total_row = {
|
||||||
'分类': '总计',
|
'分类': '总计',
|
||||||
'Benchmark': '',
|
'Benchmark': '',
|
||||||
'得分': round(total_score, 4),
|
'得分': round(total_score, 4),
|
||||||
'实测时间(h)': round(total_time, 4),
|
'实测时间(h)': round(total_time, 4),
|
||||||
|
'总样本数': total_samples if not np.isnan(total_samples) else np.nan,
|
||||||
|
'累计总tokens': total_tokens if not np.isnan(total_tokens) else np.nan,
|
||||||
}
|
}
|
||||||
for col in OUTPUT_COLUMNS:
|
for col in OUTPUT_COLUMNS:
|
||||||
if col not in total_row:
|
if col not in total_row:
|
||||||
|
|||||||
@ -151,8 +151,12 @@ SUITES = {
|
|||||||
'official': {
|
'official': {
|
||||||
'multi': ['aime25', 'aime26', 'live_code_bench'],
|
'multi': ['aime25', 'aime26', 'live_code_bench'],
|
||||||
'single': [
|
'single': [
|
||||||
'hle', 'mmlu_pro', 'gpqa_diamond', 'longbench_v2',
|
|
||||||
# 'swe_bench_verified',
|
'hle',
|
||||||
|
'mmlu_pro',
|
||||||
|
'gpqa_diamond',
|
||||||
|
'longbench_v2',
|
||||||
|
'swe_bench_verified',
|
||||||
],
|
],
|
||||||
'agent': [
|
'agent': [
|
||||||
'tau2_bench'
|
'tau2_bench'
|
||||||
|
|||||||
659
bash/test.py
659
bash/test.py
@ -1,659 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""
|
|
||||||
Unified benchmark runner for EvalScope.
|
|
||||||
|
|
||||||
A single entry point for lite / mid / full / group1 / group2 / group3 evaluations.
|
|
||||||
All tunable parameters can be controlled via command-line arguments.
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
# Full evaluation (all benchmarks, multi-run for stability)
|
|
||||||
python bash/run.py \
|
|
||||||
--model DeepSeek-V4-Flash-Int8 \
|
|
||||||
--api-url http://localhost:30000/v1 \
|
|
||||||
--dataset-dir /data1/sora/evalscope \
|
|
||||||
--output-dir /data1/sora/evalscope/output \
|
|
||||||
--suite full \
|
|
||||||
--limit none
|
|
||||||
|
|
||||||
# Lite smoke test (~5h with full samples)
|
|
||||||
python bash/run.py --suite lite --limit none
|
|
||||||
|
|
||||||
# Run only selected benchmarks
|
|
||||||
python bash/run.py --datasets aime24,gsm8k,arc --limit 20
|
|
||||||
|
|
||||||
# Custom judge model
|
|
||||||
python bash/run.py \
|
|
||||||
--judge-model deepseek-v4-pro \
|
|
||||||
--judge-api-url https://api.deepseek.com/v1 \
|
|
||||||
--judge-api-key sk-xxx
|
|
||||||
"""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from copy import deepcopy
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
from evalscope import run_task, TaskConfig
|
|
||||||
from evalscope.api.agent import NativeAgentConfig
|
|
||||||
from evalscope.config import SandboxTaskConfig
|
|
||||||
|
|
||||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
|
||||||
PROJECT_ROOT = SCRIPT_DIR.parent
|
|
||||||
|
|
||||||
# Make collect_results importable
|
|
||||||
sys.path.insert(0, str(SCRIPT_DIR))
|
|
||||||
import collect_results as collect_results_module
|
|
||||||
import perf_backup as perf_backup_module
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Default configuration (override via CLI)
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8'
|
|
||||||
DEFAULT_API_URL = 'http://localhost:30000/v1'
|
|
||||||
DEFAULT_DATASET_DIR = str(PROJECT_ROOT)
|
|
||||||
DEFAULT_OUTPUT_DIR = str(PROJECT_ROOT / 'output')
|
|
||||||
DEFAULT_CONFIG = str(PROJECT_ROOT / 'config' / 'dpv4-int8_nothinking.yaml')
|
|
||||||
DEFAULT_TOKENIZER_PATH = '/data1/models/DeepSeek-V4-Flash-INT8'
|
|
||||||
DEFAULT_LIMIT = None
|
|
||||||
DEFAULT_SEED = 42
|
|
||||||
DEFAULT_BATCH_SIZE = 4
|
|
||||||
DEFAULT_ENABLE_THINKING = False
|
|
||||||
|
|
||||||
DEFAULT_JUDGE_MODEL = 'DeepSeek/DeepSeek-V4-Pro'
|
|
||||||
DEFAULT_JUDGE_API_URL = 'https://api.vectron.meta-stone.com/v1'
|
|
||||||
DEFAULT_JUDGE_API_KEY = 'sk-dbd8a665f7634081b87ec409c7636500'
|
|
||||||
DEFAULT_JUDGE_MAX_TOKENS = 10240
|
|
||||||
|
|
||||||
# 长文本 middle-truncation 上限（token 数）。当前默认 128k。
|
|
||||||
DEFAULT_TRUNCATION_TOKENS = 32768 * 4
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Benchmark suites
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
# 多次采样配置：总样本数控制在 ~400-500
|
|
||||||
MULTI_RUN_CONFIG = {
|
|
||||||
'aime24': 12,
|
|
||||||
'aime25': 12,
|
|
||||||
'aime26': 12,
|
|
||||||
'hmmt26': 12,
|
|
||||||
'live_code_bench': 5,
|
|
||||||
'imo_answerbench': 4,
|
|
||||||
'humaneval': 3,
|
|
||||||
'gpqa_diamond': 2,
|
|
||||||
}
|
|
||||||
|
|
||||||
# 能力域完整列表
|
|
||||||
ALL_MULTI_RUN = [
|
|
||||||
# 'humaneval', 'live_code_bench',
|
|
||||||
# 'aime24', 'aime25', 'aime26', 'hmmt26',
|
|
||||||
# 'imo_answerbench', 'gpqa_diamond',
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
ALL_SINGLE_RUN = [
|
|
||||||
# 'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa',
|
|
||||||
# 'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro',
|
|
||||||
# 'simple_qa', 'trivia_qa', 'winogrande',
|
|
||||||
# 'openai_mrcr', 'longbench_v2',
|
|
||||||
'swe_bench_verified', 'swe_bench_pro',
|
|
||||||
]
|
|
||||||
ALL_AGENT = [
|
|
||||||
# 'tau2_bench', 'general_fc'
|
|
||||||
]
|
|
||||||
|
|
||||||
# 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡：
|
|
||||||
# Group1: ~61h | Group2: ~62h | Group3: ~55h
|
|
||||||
SUITES = {
|
|
||||||
'full': {
|
|
||||||
'multi': ALL_MULTI_RUN,
|
|
||||||
'single': ALL_SINGLE_RUN,
|
|
||||||
'agent': ALL_AGENT,
|
|
||||||
},
|
|
||||||
'lite': {
|
|
||||||
'multi': ['aime24', 'humaneval'],
|
|
||||||
'single': ['gsm8k', 'arc', 'longbench_v2'],
|
|
||||||
'agent': ['general_fc'],
|
|
||||||
},
|
|
||||||
'mid': {
|
|
||||||
'multi': ['aime24', 'humaneval'],
|
|
||||||
'single': [
|
|
||||||
'live_code_bench', 'bigcodebench', 'competition_math', 'gsm8k',
|
|
||||||
'gpqa_diamond', 'mmlu_pro', 'simple_qa', 'longbench_v2', 'openai_mrcr',
|
|
||||||
],
|
|
||||||
'agent': ['general_fc', 'tau2_bench'],
|
|
||||||
},
|
|
||||||
# 多机组分组，基于 CSV 实测完整时间（已含 multi-run）平衡：
|
|
||||||
# Group1: ~22.7h | Group2: ~25.6h | Group3: ~27.0h | 合计 ~75.2h
|
|
||||||
'group1': {
|
|
||||||
'multi': ['live_code_bench', 'aime24', 'aime25', 'aime26', 'hmmt26', 'imo_answerbench', 'humaneval'],
|
|
||||||
'single': ['bigcodebench', 'competition_math', 'gsm8k', 'drop', 'arc', 'hellaswag', 'winogrande'],
|
|
||||||
'agent': [],
|
|
||||||
},
|
|
||||||
'group2': {
|
|
||||||
'multi': [],
|
|
||||||
'single': ['hle', 'mmlu_pro', 'trivia_qa'],
|
|
||||||
'agent': [],
|
|
||||||
},
|
|
||||||
'group3': {
|
|
||||||
'multi': ['gpqa_diamond'],
|
|
||||||
'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'],
|
|
||||||
'agent': ['tau2_bench', 'general_fc'],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Fixed configuration
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
MATH_DATASETS = {
|
|
||||||
'aime24', 'aime25', 'aime26', 'hmmt26',
|
|
||||||
'gsm8k', 'competition_math', 'imo_answerbench',
|
|
||||||
}
|
|
||||||
|
|
||||||
MATH_PROMPT_TEMPLATE = (
|
|
||||||
"{question}\n"
|
|
||||||
"Please reason step by step, and put your final answer within \\boxed{{}}."
|
|
||||||
)
|
|
||||||
|
|
||||||
SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'swe_bench_verified', 'swe_bench_pro'}
|
|
||||||
SANDBOX_CONFIGS = {
|
|
||||||
'bigcodebench': {
|
|
||||||
'image': 'bigcodebench-sandbox:latest',
|
|
||||||
'working_dir': '/tmp',
|
|
||||||
'tools_config': {
|
|
||||||
'shell_executor': {},
|
|
||||||
'python_executor': {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'humaneval': {
|
|
||||||
'image': 'python:3.11-slim',
|
|
||||||
'tools_config': {
|
|
||||||
'shell_executor': {},
|
|
||||||
'python_executor': {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'swe_bench_verified': {
|
|
||||||
'image': 'swe-bench-sandbox:latest',
|
|
||||||
'working_dir': '/tmp',
|
|
||||||
'tools_config': {
|
|
||||||
'shell_executor': {},
|
|
||||||
'python_executor': {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'swe_bench_pro': {
|
|
||||||
'image': 'swe-bench-sandbox:latest',
|
|
||||||
'working_dir': '/tmp',
|
|
||||||
'tools_config': {
|
|
||||||
'shell_executor': {},
|
|
||||||
'python_executor': {}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# CLI parser
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def build_parser():
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description='Unified EvalScope benchmark runner',
|
|
||||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
||||||
epilog='Suites: full, lite, mid, group1, group2, group3',
|
|
||||||
)
|
|
||||||
|
|
||||||
# Model / API
|
|
||||||
parser.add_argument('--model', default=DEFAULT_MODEL,
|
|
||||||
help='Served model name (default: %(default)s)')
|
|
||||||
parser.add_argument('--api-url', default=DEFAULT_API_URL,
|
|
||||||
help='OpenAI-compatible API URL (default: %(default)s)')
|
|
||||||
|
|
||||||
# Paths
|
|
||||||
parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR,
|
|
||||||
help='Parent directory containing datasets/ subdir (default: %(default)s)')
|
|
||||||
parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR,
|
|
||||||
help='Output root directory (default: %(default)s)')
|
|
||||||
parser.add_argument('--config', default=DEFAULT_CONFIG,
|
|
||||||
help='YAML config path (default: %(default)s)')
|
|
||||||
parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH,
|
|
||||||
help='Local tokenizer path for middle-truncation (default: %(default)s)')
|
|
||||||
|
|
||||||
# Run control
|
|
||||||
parser.add_argument('--suite', default='full', choices=list(SUITES.keys()),
|
|
||||||
help='Benchmark suite to run (default: %(default)s)')
|
|
||||||
parser.add_argument('--datasets', '--benchmarks', dest='datasets', default=None,
|
|
||||||
help='Override suite with comma-separated benchmark names, e.g. aime24,gsm8k')
|
|
||||||
parser.add_argument('--exclude', default=None,
|
|
||||||
help='Comma-separated benchmarks to exclude from the chosen suite')
|
|
||||||
parser.add_argument('--limit', default=None,
|
|
||||||
help='Max samples per benchmark; "none"/"all" for no limit (default: none)')
|
|
||||||
parser.add_argument('--seed', type=int, default=DEFAULT_SEED,
|
|
||||||
help='Random seed (default: %(default)s)')
|
|
||||||
parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE,
|
|
||||||
help='Evaluation batch size (default: %(default)s)')
|
|
||||||
|
|
||||||
# Decoding / thinking
|
|
||||||
parser.add_argument('--thinking', action='store_true', default=None,
|
|
||||||
help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)')
|
|
||||||
parser.add_argument('--no-thinking', dest='thinking', action='store_false',
|
|
||||||
help='Disable thinking mode (default)')
|
|
||||||
|
|
||||||
# Judge model
|
|
||||||
parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL,
|
|
||||||
help='Judge model name (default: %(default)s)')
|
|
||||||
parser.add_argument('--judge-api-url', default=DEFAULT_JUDGE_API_URL,
|
|
||||||
help='Judge model API URL (default: %(default)s)')
|
|
||||||
parser.add_argument('--judge-api-key', default=DEFAULT_JUDGE_API_KEY,
|
|
||||||
help='Judge model API key')
|
|
||||||
parser.add_argument('--judge-max-tokens', type=int, default=DEFAULT_JUDGE_MAX_TOKENS,
|
|
||||||
help='Judge model max_tokens (default: %(default)s)')
|
|
||||||
|
|
||||||
# Truncation
|
|
||||||
parser.add_argument('--truncation-tokens', type=int, default=DEFAULT_TRUNCATION_TOKENS,
|
|
||||||
help='Middle-truncation token budget for long-context benchmarks (default: %(default)s)')
|
|
||||||
|
|
||||||
# Result collection
|
|
||||||
parser.add_argument('--no-summary', dest='write_summary', action='store_false',
|
|
||||||
help='Skip writing summary Excel/CSV after each benchmark')
|
|
||||||
parser.add_argument('--summary-name', default=None,
|
|
||||||
help='Output summary file name (without extension); defaults to safe model name')
|
|
||||||
|
|
||||||
return parser
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Middle-truncation helpers
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
_TOKENIZER = None
|
|
||||||
|
|
||||||
|
|
||||||
def get_tokenizer(tokenizer_path: str):
|
|
||||||
global _TOKENIZER
|
|
||||||
if _TOKENIZER is None:
|
|
||||||
from transformers import AutoTokenizer
|
|
||||||
try:
|
|
||||||
_TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True)
|
|
||||||
except Exception:
|
|
||||||
_TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True)
|
|
||||||
return _TOKENIZER
|
|
||||||
|
|
||||||
|
|
||||||
def truncate_middle(text: str, max_tokens: int, tokenizer_path: str) -> str:
|
|
||||||
if max_tokens <= 0:
|
|
||||||
return text
|
|
||||||
tokenizer = get_tokenizer(tokenizer_path)
|
|
||||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
|
||||||
if len(token_ids) <= max_tokens:
|
|
||||||
return text
|
|
||||||
keep_head = max_tokens // 2
|
|
||||||
keep_tail = max_tokens - keep_head
|
|
||||||
truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:]
|
|
||||||
return tokenizer.decode(truncated_ids, skip_special_tokens=True)
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_adapters_for_truncation(tokenizer_path: str, truncation_tokens: int):
|
|
||||||
from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter
|
|
||||||
from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter
|
|
||||||
|
|
||||||
_orig_longbench_format = LongBenchV2Adapter.format_prompt_template
|
|
||||||
|
|
||||||
def _patched_longbench_format(self, sample):
|
|
||||||
if sample.metadata and 'context' in sample.metadata:
|
|
||||||
sample.metadata['context'] = truncate_middle(sample.metadata['context'], truncation_tokens, tokenizer_path)
|
|
||||||
return _orig_longbench_format(self, sample)
|
|
||||||
|
|
||||||
LongBenchV2Adapter.format_prompt_template = _patched_longbench_format
|
|
||||||
|
|
||||||
_orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample
|
|
||||||
|
|
||||||
def _patched_mrcr_record(self, record):
|
|
||||||
per_msg_max_tok = 8192
|
|
||||||
if 'prompt' in record:
|
|
||||||
try:
|
|
||||||
prompt_data = json.loads(record['prompt'])
|
|
||||||
if not isinstance(prompt_data, list) or len(prompt_data) == 0:
|
|
||||||
return _orig_mrcr_record(self, record)
|
|
||||||
|
|
||||||
tokenizer = get_tokenizer(tokenizer_path)
|
|
||||||
total_tok = sum(
|
|
||||||
len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False))
|
|
||||||
for msg in prompt_data
|
|
||||||
)
|
|
||||||
if total_tok <= truncation_tokens:
|
|
||||||
return _orig_mrcr_record(self, record)
|
|
||||||
|
|
||||||
desired_idx = record.get('desired_msg_index', 0)
|
|
||||||
if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data):
|
|
||||||
desired_idx = 0
|
|
||||||
|
|
||||||
n = len(prompt_data)
|
|
||||||
keep = set()
|
|
||||||
keep.update(range(min(2, n)))
|
|
||||||
keep.update(range(max(0, n - 2), n))
|
|
||||||
window = 2
|
|
||||||
keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1)))
|
|
||||||
keep = sorted(keep)
|
|
||||||
|
|
||||||
new_prompt = []
|
|
||||||
for idx in keep:
|
|
||||||
msg = prompt_data[idx]
|
|
||||||
if isinstance(msg, dict):
|
|
||||||
msg = dict(msg)
|
|
||||||
content = msg.get('content', '')
|
|
||||||
if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok:
|
|
||||||
msg['content'] = truncate_middle(content, per_msg_max_tok, tokenizer_path)
|
|
||||||
new_prompt.append(msg)
|
|
||||||
|
|
||||||
record = dict(record)
|
|
||||||
record['prompt'] = json.dumps(new_prompt)
|
|
||||||
except (json.JSONDecodeError, TypeError):
|
|
||||||
pass
|
|
||||||
return _orig_mrcr_record(self, record)
|
|
||||||
|
|
||||||
OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Helpers
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def load_dataset_configs(config_path: str):
|
|
||||||
if not Path(config_path).exists():
|
|
||||||
raise FileNotFoundError(f'Config file not found: {config_path}')
|
|
||||||
with open(config_path, 'r', encoding='utf-8') as f:
|
|
||||||
return yaml.safe_load(f)
|
|
||||||
|
|
||||||
|
|
||||||
def configure_thinking(generation_config: dict, enable: bool) -> dict:
|
|
||||||
extra_body = generation_config.get('extra_body', {})
|
|
||||||
chat_template_kwargs = extra_body.get('chat_template_kwargs', {})
|
|
||||||
if enable:
|
|
||||||
chat_template_kwargs['thinking'] = True
|
|
||||||
else:
|
|
||||||
chat_template_kwargs.pop('thinking', None)
|
|
||||||
if chat_template_kwargs:
|
|
||||||
extra_body['chat_template_kwargs'] = chat_template_kwargs
|
|
||||||
if extra_body:
|
|
||||||
generation_config['extra_body'] = extra_body
|
|
||||||
return generation_config
|
|
||||||
|
|
||||||
|
|
||||||
def build_agent_config(agent_cfg: dict) -> NativeAgentConfig:
|
|
||||||
agent_cfg = deepcopy(agent_cfg or {})
|
|
||||||
known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'}
|
|
||||||
kwargs = agent_cfg.pop('kwargs', {})
|
|
||||||
for key in list(agent_cfg.keys()):
|
|
||||||
if key not in known_fields:
|
|
||||||
kwargs[key] = agent_cfg.pop(key)
|
|
||||||
if kwargs:
|
|
||||||
agent_cfg['kwargs'] = kwargs
|
|
||||||
return NativeAgentConfig(**agent_cfg)
|
|
||||||
|
|
||||||
|
|
||||||
def build_task_config(
|
|
||||||
dataset_name: str,
|
|
||||||
ds_cfg: dict,
|
|
||||||
batch_size: int,
|
|
||||||
enable_thinking: bool,
|
|
||||||
seed: int,
|
|
||||||
limit,
|
|
||||||
output_dir: str,
|
|
||||||
model: str,
|
|
||||||
api_url: str,
|
|
||||||
dataset_dir: str,
|
|
||||||
judge_model_args: dict,
|
|
||||||
run_idx: int = 0,
|
|
||||||
) -> TaskConfig:
|
|
||||||
if run_idx > 0:
|
|
||||||
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
|
|
||||||
else:
|
|
||||||
work_dir = Path(output_dir) / dataset_name / f'seed_{seed}'
|
|
||||||
work_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
work_dir = str(work_dir)
|
|
||||||
|
|
||||||
generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking)
|
|
||||||
dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
|
|
||||||
dataset_args.setdefault('shuffle', True)
|
|
||||||
|
|
||||||
if dataset_name in MATH_DATASETS:
|
|
||||||
dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE
|
|
||||||
|
|
||||||
dataset_args_dict = {dataset_name: dataset_args}
|
|
||||||
|
|
||||||
agent_config = None
|
|
||||||
if 'agent_config' in ds_cfg:
|
|
||||||
agent_config = build_agent_config(ds_cfg['agent_config'])
|
|
||||||
|
|
||||||
return TaskConfig(
|
|
||||||
model=model,
|
|
||||||
api_url=api_url,
|
|
||||||
eval_type='openai_api',
|
|
||||||
dataset_dir=dataset_dir,
|
|
||||||
judge_model_args=judge_model_args,
|
|
||||||
seed=seed,
|
|
||||||
limit=limit,
|
|
||||||
collect_perf=True,
|
|
||||||
no_timestamp=True,
|
|
||||||
work_dir=work_dir,
|
|
||||||
use_cache=work_dir,
|
|
||||||
datasets=[dataset_name],
|
|
||||||
generation_config=generation_config,
|
|
||||||
dataset_args=dataset_args_dict,
|
|
||||||
agent_config=agent_config,
|
|
||||||
eval_batch_size=batch_size,
|
|
||||||
sandbox=SandboxTaskConfig(
|
|
||||||
enabled=True,
|
|
||||||
engine='docker',
|
|
||||||
default_config=SANDBOX_CONFIGS.get(dataset_name, {
|
|
||||||
'image': 'python:3.11-slim',
|
|
||||||
'tools_config': {
|
|
||||||
'shell_executor': {},
|
|
||||||
'python_executor': {}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
) if dataset_name in SANDBOX_DATASETS else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Main
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def write_summary(output_dir: str, model_name: str, summary_name: str):
|
|
||||||
"""Re-aggregate results across all benchmarks under output_dir."""
|
|
||||||
try:
|
|
||||||
excel_output_dir = PROJECT_ROOT / 'results'
|
|
||||||
collect_results_module.collect_all(
|
|
||||||
Path(output_dir), model_name, summary_name,
|
|
||||||
excel_output_dir=excel_output_dir,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print(f'WARNING: failed to write summary: {e}')
|
|
||||||
|
|
||||||
|
|
||||||
def backup_after_run(output_dir: str, benchmark: str, model_name: str,
|
|
||||||
work_dir: Path):
|
|
||||||
"""Snapshot the just-finished run's perf summary and predictions."""
|
|
||||||
try:
|
|
||||||
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
|
|
||||||
perf_backup_module.backup_perf_stats(
|
|
||||||
Path(output_dir), benchmark, model_name, report_json,
|
|
||||||
)
|
|
||||||
predictions_dir = work_dir / 'predictions' / model_name
|
|
||||||
perf_backup_module.archive_predictions(
|
|
||||||
Path(output_dir), benchmark, model_name, predictions_dir,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print(f'WARNING: perf backup for {benchmark} failed: {e}')
|
|
||||||
|
|
||||||
|
|
||||||
def restore_before_run(output_dir: str, benchmark: str, model_name: str,
|
|
||||||
work_dir: Path) -> bool:
|
|
||||||
"""If durable backups exist for a benchmark/model, materialise them into
|
|
||||||
the new ``work_dir`` before evalscope starts so the next run resumes from
|
|
||||||
the larger historical state instead of overwriting it.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
predictions_dir = work_dir / 'predictions' / model_name
|
|
||||||
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
|
|
||||||
restored = perf_backup_module.restore_from_backup(
|
|
||||||
Path(output_dir), benchmark, model_name,
|
|
||||||
predictions_dir, report_json,
|
|
||||||
)
|
|
||||||
if restored:
|
|
||||||
print(f'Restored {benchmark}/{model_name} from backup before run')
|
|
||||||
return restored
|
|
||||||
except Exception as e:
|
|
||||||
print(f'WARNING: perf restore for {benchmark} failed: {e}')
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, model_name: str, summary_name: str):
|
|
||||||
"""Run a benchmark task and optionally refresh the summary table.
|
|
||||||
|
|
||||||
After a successful ``run_task()`` we snapshot the cumulative
|
|
||||||
``perf_metrics.summary`` and the per-sample predictions into the durable
|
|
||||||
backup files maintained by ``perf_backup.py`` so checkpoint restarts can
|
|
||||||
recover them.
|
|
||||||
"""
|
|
||||||
dataset_name = task_cfg.datasets[0]
|
|
||||||
work_dir = Path(task_cfg.work_dir)
|
|
||||||
restore_before_run(output_dir, dataset_name, model_name, work_dir)
|
|
||||||
|
|
||||||
start_ts = time.monotonic()
|
|
||||||
try:
|
|
||||||
run_task(task_cfg)
|
|
||||||
finally:
|
|
||||||
elapsed = time.monotonic() - start_ts
|
|
||||||
perf_backup_module.record_active_time(output_dir, dataset_name, model_name, elapsed)
|
|
||||||
print(f'Active time for {dataset_name}: {elapsed:.1f}s (total accumulated)')
|
|
||||||
|
|
||||||
backup_after_run(output_dir, dataset_name, model_name, work_dir)
|
|
||||||
if write_summary_flag:
|
|
||||||
write_summary(output_dir, model_name, summary_name)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
parser = build_parser()
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
# Resolve limit
|
|
||||||
limit = args.limit
|
|
||||||
if limit is not None:
|
|
||||||
if str(limit).lower() in ('none', 'all'):
|
|
||||||
limit = None
|
|
||||||
else:
|
|
||||||
limit = int(limit)
|
|
||||||
|
|
||||||
enable_thinking = DEFAULT_ENABLE_THINKING if args.thinking is None else args.thinking
|
|
||||||
|
|
||||||
# Resolve suite or custom datasets
|
|
||||||
if args.datasets:
|
|
||||||
custom = [d.strip() for d in args.datasets.split(',') if d.strip()]
|
|
||||||
multi_run = [d for d in custom if d in MULTI_RUN_CONFIG]
|
|
||||||
single_run = [d for d in custom if d not in MULTI_RUN_CONFIG]
|
|
||||||
agent = [d for d in custom if d in ALL_AGENT]
|
|
||||||
single_run = [d for d in single_run if d not in ALL_AGENT]
|
|
||||||
else:
|
|
||||||
suite = SUITES[args.suite]
|
|
||||||
multi_run = list(suite['multi'])
|
|
||||||
single_run = list(suite['single'])
|
|
||||||
agent = list(suite['agent'])
|
|
||||||
|
|
||||||
# Apply --exclude
|
|
||||||
if args.exclude:
|
|
||||||
exclude = {d.strip() for d in args.exclude.split(',') if d.strip()}
|
|
||||||
multi_run = [d for d in multi_run if d not in exclude]
|
|
||||||
single_run = [d for d in single_run if d not in exclude]
|
|
||||||
agent = [d for d in agent if d not in exclude]
|
|
||||||
|
|
||||||
judge_model_args = {
|
|
||||||
'model_id': args.judge_model,
|
|
||||||
'api_url': args.judge_api_url,
|
|
||||||
'api_key': args.judge_api_key,
|
|
||||||
'eval_type': 'openai_api',
|
|
||||||
'generation_config': {
|
|
||||||
'temperature': 0.0,
|
|
||||||
'max_tokens': args.judge_max_tokens,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
truncation_tokens = args.truncation_tokens
|
|
||||||
_patch_adapters_for_truncation(args.tokenizer_path, truncation_tokens)
|
|
||||||
|
|
||||||
dataset_configs = load_dataset_configs(args.config)
|
|
||||||
|
|
||||||
print('=' * 60)
|
|
||||||
print(f'Config: {args.config}')
|
|
||||||
print(f'Model: {args.model}')
|
|
||||||
print(f'API URL: {args.api_url}')
|
|
||||||
print(f'Dataset Dir: {args.dataset_dir}')
|
|
||||||
print(f'Output Dir: {args.output_dir}')
|
|
||||||
print(f'Suite: {args.suite}')
|
|
||||||
print(f'Limit: {limit if limit is not None else "ALL"}')
|
|
||||||
print(f'Thinking: {enable_thinking}')
|
|
||||||
print(f'Seed: {args.seed}')
|
|
||||||
print(f'Batch Size: {args.batch_size}')
|
|
||||||
print(f'Tokenizer Path: {args.tokenizer_path}')
|
|
||||||
print(f'Truncation Tokens: {truncation_tokens}')
|
|
||||||
print(f'Multi-run datasets: {multi_run}')
|
|
||||||
print(f'Single-run datasets: {single_run}')
|
|
||||||
print(f'Agent datasets: {agent}')
|
|
||||||
print(f'Write summary: {args.write_summary}')
|
|
||||||
print('=' * 60)
|
|
||||||
|
|
||||||
def run_one(dataset_name, run_idx=0):
|
|
||||||
ds_cfg = dataset_configs[dataset_name]
|
|
||||||
task_cfg = build_task_config(
|
|
||||||
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
|
|
||||||
args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
|
|
||||||
run_idx=run_idx,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, args.summary_name)
|
|
||||||
except Exception as e:
|
|
||||||
print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
|
|
||||||
|
|
||||||
for dataset_name in multi_run:
|
|
||||||
if dataset_name not in dataset_configs:
|
|
||||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
|
||||||
continue
|
|
||||||
num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1)
|
|
||||||
for run_idx in range(num_runs):
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})')
|
|
||||||
print(f"{'='*60}")
|
|
||||||
run_one(dataset_name, run_idx=run_idx)
|
|
||||||
|
|
||||||
for dataset_name in single_run:
|
|
||||||
if dataset_name not in dataset_configs:
|
|
||||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
|
||||||
continue
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f'Running: {dataset_name} (seed={args.seed})')
|
|
||||||
print(f"{'='*60}")
|
|
||||||
run_one(dataset_name)
|
|
||||||
|
|
||||||
for dataset_name in agent:
|
|
||||||
if dataset_name not in dataset_configs:
|
|
||||||
print(f'WARNING: {dataset_name} not in YAML config, skipping')
|
|
||||||
continue
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f'Running: {dataset_name} (seed={args.seed})')
|
|
||||||
print(f"{'='*60}")
|
|
||||||
run_one(dataset_name)
|
|
||||||
|
|
||||||
if args.write_summary:
|
|
||||||
write_summary(args.output_dir, args.model, args.summary_name)
|
|
||||||
print('\nAll benchmarks done!')
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
179
bash/test_e2e.py
179
bash/test_e2e.py
@ -1,179 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""End-to-end offline test for bash/run.py + collect_results + perf_backup.
|
|
||||||
|
|
||||||
This script does NOT call any model API. It uses an existing benchmark
|
|
||||||
directory (aime24/seed_42) and walks through every code path that the real
|
|
||||||
``run.py`` would touch:
|
|
||||||
|
|
||||||
1. backup_perf_stats — confirm a perf summary snapshot is written
|
|
||||||
2. archive_predictions — confirm per-sample records are deduplicated into
|
|
||||||
the archive
|
|
||||||
3. restore_from_backup — confirm a fresh work_dir can be rehydrated from
|
|
||||||
the backup files
|
|
||||||
4. collect_results aggregation — confirm correct metrics are produced
|
|
||||||
5. Whitelist mode — confirm only the requested benchmarks appear in the
|
|
||||||
summary
|
|
||||||
6. hle_low alias — confirm hle_low directory maps to canonical "hle"
|
|
||||||
7. Reset recovery — confirm that even after wiping the predictions file
|
|
||||||
and the report summary, the archive+backup still recover the metrics
|
|
||||||
|
|
||||||
Run from the project root:
|
|
||||||
|
|
||||||
python bash/test_e2e.py
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
|
||||||
import shutil
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent))
|
|
||||||
|
|
||||||
import collect_results as cr # noqa: E402
|
|
||||||
import perf_backup as pb # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
PROJECT_ROOT = Path('/data1/sora/evalscope')
|
|
||||||
SOURCE_OUTPUT = PROJECT_ROOT / 'output'
|
|
||||||
|
|
||||||
|
|
||||||
def banner(msg):
|
|
||||||
print('\n' + '=' * 60)
|
|
||||||
print(f' {msg}')
|
|
||||||
print('=' * 60)
|
|
||||||
|
|
||||||
|
|
||||||
def assert_close(actual, expected, name, tol=1e-3):
|
|
||||||
if abs(actual - expected) > tol:
|
|
||||||
raise AssertionError(f'{name}: expected ~{expected}, got {actual}')
|
|
||||||
print(f' OK {name}={actual:.4f}')
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
banner('Setting up isolated test output dir')
|
|
||||||
test_root = PROJECT_ROOT / 'output_e2e_test'
|
|
||||||
if test_root.exists():
|
|
||||||
shutil.rmtree(test_root)
|
|
||||||
test_root.mkdir(parents=True)
|
|
||||||
|
|
||||||
# Copy the canonical aime24 data into the isolated dir
|
|
||||||
src = SOURCE_OUTPUT / 'aime24' / 'seed_42'
|
|
||||||
dst = test_root / 'aime24' / 'seed_42'
|
|
||||||
shutil.copytree(src, dst)
|
|
||||||
|
|
||||||
# And copy hle_low as hle_low under the test dir (so we exercise the
|
|
||||||
# alias code path).
|
|
||||||
hle_src = SOURCE_OUTPUT / 'hle_low' / 'seed_42'
|
|
||||||
hle_dst = test_root / 'hle_low' / 'seed_42'
|
|
||||||
if hle_src.exists():
|
|
||||||
shutil.copytree(hle_src, hle_dst)
|
|
||||||
|
|
||||||
benchmark = 'aime24'
|
|
||||||
model_name = 'DeepSeek-V4-Flash-Int8'
|
|
||||||
|
|
||||||
banner('1) backup_perf_stats — snapshot summary')
|
|
||||||
report_json = dst / 'reports' / model_name / f'{benchmark}.json'
|
|
||||||
backup_path = pb.backup_perf_stats(test_root, benchmark, model_name, report_json)
|
|
||||||
assert backup_path.exists(), 'backup file was not created'
|
|
||||||
payload = json.loads(backup_path.read_text())
|
|
||||||
assert payload['n_samples'] == 30, f'expected 30 samples, got {payload["n_samples"]}'
|
|
||||||
print(f' OK backup written: {backup_path}, n_samples={payload["n_samples"]}')
|
|
||||||
|
|
||||||
banner('2) archive_predictions — deduplicate into archive')
|
|
||||||
preds = dst / 'predictions' / model_name
|
|
||||||
archive_path = pb.archive_predictions(test_root, benchmark, model_name, preds)
|
|
||||||
assert archive_path.exists(), 'archive file was not created'
|
|
||||||
n_lines = sum(1 for line in archive_path.read_text().splitlines() if line.strip())
|
|
||||||
assert n_lines == 30, f'expected 30 archived samples, got {n_lines}'
|
|
||||||
print(f' OK archive written: {archive_path}, samples={n_lines}')
|
|
||||||
|
|
||||||
banner('3) restore_from_backup — rehydrate a fresh work_dir')
|
|
||||||
fresh_dir = test_root / 'fresh_work' / 'seed_42'
|
|
||||||
fresh_report = fresh_dir / 'reports' / model_name / f'{benchmark}.json'
|
|
||||||
fresh_preds = fresh_dir / 'predictions' / model_name
|
|
||||||
# Pre-create empty structure to mimic what eval_task would do
|
|
||||||
fresh_report.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
fresh_report.write_text(json.dumps({
|
|
||||||
'name': f'{model_name}@{benchmark}',
|
|
||||||
'perf_metrics': {'summary': {'n_samples': 1}},
|
|
||||||
}))
|
|
||||||
restored = pb.restore_from_backup(
|
|
||||||
test_root, benchmark, model_name, fresh_preds, fresh_report,
|
|
||||||
)
|
|
||||||
assert restored, 'restore_from_backup returned False'
|
|
||||||
restored_report = json.loads(fresh_report.read_text())
|
|
||||||
assert restored_report['perf_metrics']['summary']['n_samples'] == 30, \
|
|
||||||
f'restored n_samples={restored_report["perf_metrics"]["summary"]["n_samples"]}'
|
|
||||||
print(' OK restored report n_samples=30 (matches backup)')
|
|
||||||
|
|
||||||
banner('4) collect_results — aggregate metrics')
|
|
||||||
csv, xlsx = cr.eval_benchmark([benchmark], test_root, model_name)
|
|
||||||
import pandas as pd
|
|
||||||
df = pd.read_csv(csv)
|
|
||||||
aime24 = df[df['Benchmark'] == 'aime24'].iloc[0]
|
|
||||||
# Single seed_42 run, so values match the report directly (no multi-run
|
|
||||||
# averaging). The full-summary CSV averages across run_1..run_11.
|
|
||||||
assert_close(aime24['得分'], 0.6333, 'aime24 score (single seed)')
|
|
||||||
assert_close(aime24['实测时间(h)'], 0.1469, 'aime24 duration (h)')
|
|
||||||
assert_close(aime24['总样本数'], 30, 'aime24 sample count')
|
|
||||||
assert_close(aime24['延迟_mean(s)'], 35.24971, 'aime24 latency mean')
|
|
||||||
assert_close(aime24['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean')
|
|
||||||
assert_close(aime24['TPOT_mean(s)'], 0.02165, 'aime24 TPOT mean')
|
|
||||||
assert_close(aime24['输入tokens_mean'], 119.33, 'aime24 input_tokens_mean')
|
|
||||||
assert_close(aime24['输出tokens_mean'], 1609.8, 'aime24 output_tokens_mean')
|
|
||||||
|
|
||||||
banner('5) Whitelist mode — only the requested benchmarks appear')
|
|
||||||
csv2, _ = cr.eval_benchmark([benchmark], test_root, model_name)
|
|
||||||
df2 = pd.read_csv(csv2)
|
|
||||||
assert df2['Benchmark'].dropna().tolist() == ['aime24'], \
|
|
||||||
f'whitelist did not limit benchmarks: {df2["Benchmark"].tolist()}'
|
|
||||||
print(' OK summary only contains aime24 + total')
|
|
||||||
|
|
||||||
banner('6) hle_low → hle alias')
|
|
||||||
if hle_src.exists():
|
|
||||||
csv3, _ = cr.eval_benchmark(['hle'], test_root, model_name)
|
|
||||||
df3 = pd.read_csv(csv3)
|
|
||||||
assert 'hle' in df3['Benchmark'].tolist(), \
|
|
||||||
f'hle canonical name missing from: {df3["Benchmark"].tolist()}'
|
|
||||||
assert 'hle_low' not in df3['Benchmark'].tolist(), \
|
|
||||||
f'hle_low should be aliased away: {df3["Benchmark"].tolist()}'
|
|
||||||
print(' OK hle_low directory maps to canonical "hle"')
|
|
||||||
|
|
||||||
banner('7) Reset recovery — wipe report+predictions, expect archive to restore')
|
|
||||||
# Wipe the current predictions
|
|
||||||
shutil.rmtree(preds)
|
|
||||||
# Reset report.json
|
|
||||||
data = json.loads(report_json.read_text())
|
|
||||||
data['perf_metrics']['summary']['n_samples'] = 1
|
|
||||||
data['perf_metrics']['summary']['latency']['mean'] = 0.0
|
|
||||||
report_json.write_text(json.dumps(data, indent=2))
|
|
||||||
|
|
||||||
csv4, _ = cr.eval_benchmark([benchmark], test_root, model_name)
|
|
||||||
df4 = pd.read_csv(csv4)
|
|
||||||
aime24_after = df4[df4['Benchmark'] == 'aime24'].iloc[0]
|
|
||||||
assert_close(aime24_after['得分'], 0.6333, 'aime24 score (post-reset)')
|
|
||||||
assert_close(aime24_after['总样本数'], 30, 'aime24 sample count (post-reset)')
|
|
||||||
assert_close(aime24_after['延迟_mean(s)'], 35.24971, 'aime24 latency mean (post-reset)')
|
|
||||||
assert_close(aime24_after['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean (post-reset)')
|
|
||||||
print(' OK archive fully recovered metrics after predictions wipe')
|
|
||||||
|
|
||||||
banner('8) Backup monotonicity — new backup must not overwrite a larger one')
|
|
||||||
# We have a backup with n_samples=30. Now write a fresh report with
|
|
||||||
# n_samples=2 and call backup_perf_stats again — the backup should NOT
|
|
||||||
# be overwritten.
|
|
||||||
data = json.loads(report_json.read_text())
|
|
||||||
data['perf_metrics']['summary']['n_samples'] = 2
|
|
||||||
report_json.write_text(json.dumps(data, indent=2))
|
|
||||||
pb.backup_perf_stats(test_root, benchmark, model_name, report_json)
|
|
||||||
payload2 = json.loads(backup_path.read_text())
|
|
||||||
assert payload2['n_samples'] == 30, \
|
|
||||||
f'backup was clobbered: n_samples now {payload2["n_samples"]}'
|
|
||||||
print(' OK backup preserved n_samples=30 even after a reset run')
|
|
||||||
|
|
||||||
banner('All checks passed')
|
|
||||||
print(f'Test artifacts under {test_root} (kept for inspection)')
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -101,6 +101,28 @@ def eval_instance(
|
|||||||
log_dir = Path(log_dir) / 'swebench_log' / instance_id
|
log_dir = Path(log_dir) / 'swebench_log' / instance_id
|
||||||
|
|
||||||
log_dir.mkdir(parents=True, exist_ok=True)
|
log_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Resume: skip container evaluation if test output already exists.
|
||||||
|
test_output_path = log_dir / LOG_TEST_OUTPUT
|
||||||
|
if test_output_path.exists():
|
||||||
|
logger.info(f'Found existing test output for {instance_id}, skip container evaluation and reuse log.')
|
||||||
|
try:
|
||||||
|
report = get_eval_report(
|
||||||
|
test_spec=test_spec,
|
||||||
|
prediction=pred,
|
||||||
|
test_log_path=test_output_path,
|
||||||
|
include_tests_status=True,
|
||||||
|
)
|
||||||
|
logger.info(f'report: {report}\n'
|
||||||
|
f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}")
|
||||||
|
return {
|
||||||
|
'completed': True,
|
||||||
|
'resolved': report.get(instance_id, {}).get('resolved', False),
|
||||||
|
'report': report,
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f'Failed to reuse existing test output for {instance_id}: {e}. Will re-evaluate.')
|
||||||
|
|
||||||
logger.info(f'Starting evaluation for {instance_id} in log dir {log_dir}...')
|
logger.info(f'Starting evaluation for {instance_id} in log dir {log_dir}...')
|
||||||
|
|
||||||
client = DockerClient.from_env()
|
client = DockerClient.from_env()
|
||||||
|
|||||||
110
myread.md
110
myread.md
@ -159,12 +159,65 @@ docker images | grep evalscope-complete-py312
|
|||||||
| `python:3.11-slim` | 通用代码执行环境 |
|
| `python:3.11-slim` | 通用代码执行环境 |
|
||||||
| `swebench/sweb.eval.x86_64.*` | SWE-bench 每个样本一个实例镜像 |
|
| `swebench/sweb.eval.x86_64.*` | SWE-bench 每个样本一个实例镜像 |
|
||||||
|
|
||||||
#### 2.4.1 配置 Docker（国内镜像加速 + 大容量存储路径）
|
根据是否跑 SWE-bench，选择下面两种 Docker 配置方式之一。
|
||||||
|
|
||||||
SWE-bench 镜像数量多、体积大，建议把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘：
|
#### 2.4.1 方式 A：不测 SWE-bench 的 Docker 配置
|
||||||
|
|
||||||
|
如果只跑常规 benchmark（不含 `swe_bench_*`），不需要迁移 Docker `data-root`，只需配置国内镜像加速并加载基础镜像：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 1. 停止 Docker
|
# 1. 暂停服务
|
||||||
|
sudo systemctl stop docker.socket
|
||||||
|
sudo systemctl stop docker
|
||||||
|
|
||||||
|
# 2. 配置 Docker 国内镜像加速
|
||||||
|
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||||
|
{
|
||||||
|
"registry-mirrors": [
|
||||||
|
"https://docker.m.daocloud.io",
|
||||||
|
"https://docker.1ms.run",
|
||||||
|
"https://hub.rat.dev",
|
||||||
|
"https://docker.1panel.live",
|
||||||
|
"https://dockerproxy.com",
|
||||||
|
"https://hub-mirror.c.163.com",
|
||||||
|
"https://mirror.baidubce.com",
|
||||||
|
"https://docker.mirrors.ustc.edu.cn",
|
||||||
|
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||||
|
"https://docker.nju.edu.cn",
|
||||||
|
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 3. 重启 Docker
|
||||||
|
sudo systemctl start docker
|
||||||
|
|
||||||
|
# 4. 加载 evalscope 环境镜像（含 Python 3.12 + 全部依赖）
|
||||||
|
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||||
|
|
||||||
|
# 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须）
|
||||||
|
docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||||
|
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
ENTRYPOINT []
|
||||||
|
CMD ["tail", "-f", "/dev/null"]
|
||||||
|
EOF
|
||||||
|
|
||||||
|
docker pull python:3.11-slim
|
||||||
|
```
|
||||||
|
|
||||||
|
验证：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker images | grep -E 'evalscope-complete-py312|bigcodebench|python:3.11-slim'
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2.4.2 方式 B：测 SWE-bench 的 Docker 配置（大容量存储路径）
|
||||||
|
|
||||||
|
SWE-bench 镜像数量多、体积大，必须把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘，否则默认盘很快会被占满：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. 暂停服务
|
||||||
sudo systemctl stop docker.socket
|
sudo systemctl stop docker.socket
|
||||||
sudo systemctl stop docker
|
sudo systemctl stop docker
|
||||||
sudo systemctl stop containerd
|
sudo systemctl stop containerd
|
||||||
@ -173,7 +226,7 @@ sudo systemctl stop containerd
|
|||||||
mkdir -p /data1/sora/evalscope/docker/images
|
mkdir -p /data1/sora/evalscope/docker/images
|
||||||
mkdir -p /data1/sora/evalscope/docker/containerd
|
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||||
|
|
||||||
# 3. 配置国内镜像加速（多填几个，自动轮询）
|
# 3. 配置国内镜像加速 + 大容量 data-root
|
||||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||||
{
|
{
|
||||||
"data-root": "/data1/sora/evalscope/docker/images",
|
"data-root": "/data1/sora/evalscope/docker/images",
|
||||||
@ -203,22 +256,18 @@ EOF
|
|||||||
sudo systemctl reset-failed docker.service
|
sudo systemctl reset-failed docker.service
|
||||||
sudo systemctl start containerd
|
sudo systemctl start containerd
|
||||||
sudo systemctl start docker
|
sudo systemctl start docker
|
||||||
```
|
|
||||||
|
|
||||||
#### 2.4.2 拉取/构建代码执行镜像
|
# 6. 加载 evalscope 环境镜像
|
||||||
|
docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz
|
||||||
|
|
||||||
```bash
|
# 7. 准备代码执行 sandbox 镜像
|
||||||
# bigcodebench 官方镜像
|
|
||||||
docker pull bigcodebench/bigcodebench-evaluate:latest
|
docker pull bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
|
||||||
# 构建守护态 sandbox
|
|
||||||
docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
docker build -t bigcodebench-sandbox:latest -f - . <<'EOF'
|
||||||
FROM bigcodebench/bigcodebench-evaluate:latest
|
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||||
ENTRYPOINT []
|
ENTRYPOINT []
|
||||||
CMD ["tail", "-f", "/dev/null"]
|
CMD ["tail", "-f", "/dev/null"]
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# 通用 Python 执行环境
|
|
||||||
docker pull python:3.11-slim
|
docker pull python:3.11-slim
|
||||||
```
|
```
|
||||||
|
|
||||||
@ -267,25 +316,42 @@ python bash/pull_swe_bench_images.py \
|
|||||||
> 2. 降低 `--max-workers` 到 1；
|
> 2. 降低 `--max-workers` 到 1；
|
||||||
> 3. 使用 `tmux` 挂后台运行，避免 SSH 断连导致中断。
|
> 3. 使用 `tmux` 挂后台运行，避免 SSH 断连导致中断。
|
||||||
|
|
||||||
#### 2.4.4 SWE-bench 镜像的本地备份与迁移（可选）
|
#### 2.4.4 SWE-bench 镜像的备份与迁移（可选）
|
||||||
|
|
||||||
SWE-bench 全量镜像约 500 个、总大小约 **2TB**，无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器，建议用移动硬盘或内网直接拷贝 Docker `data-root`，或使用下面脚本分批导出/导入：
|
SWE-bench Verified 全量约 500 个镜像、本地实际占用约 **160GB**。迁移到另一台机器有三种方式：
|
||||||
|
|
||||||
**源机器：打包**
|
**方式一：从 ModelScope 下载（推荐，已上传）**
|
||||||
|
|
||||||
|
`SoraAmami/swe-bench-verified-images` 已包含分批导出的 `docker save` tar.gz，目标机器直接下载加载即可：
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
# 下载
|
||||||
```
|
modelscope download \
|
||||||
|
--repo-type dataset \
|
||||||
|
--local_dir /data1/sora/evalscope/docker/swe_images \
|
||||||
|
SoraAmami/swe-bench-verified-images
|
||||||
|
|
||||||
脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。
|
# 加载
|
||||||
|
|
||||||
**目标机器：加载**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||||
```
|
```
|
||||||
|
|
||||||
> 注意：分批 tar 会占用额外磁盘空间（约 2TB），请确保目标盘容量足够。
|
**方式二：本地分批导出/导入**
|
||||||
|
|
||||||
|
如果已有本地镜像，可用脚本打包后复制到目标机器：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 源机器打包（每 50 个一批）
|
||||||
|
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
||||||
|
|
||||||
|
# 目标机器加载
|
||||||
|
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||||
|
```
|
||||||
|
|
||||||
|
**方式三：直接拷贝 Docker data-root**
|
||||||
|
|
||||||
|
适用于内网或移动硬盘，把 `/data1/sora/evalscope/docker/images` 和 `/data1/sora/evalscope/docker/containerd` 完整复制到目标机器相同路径，再启动 Docker。注意这种方式对 Docker 版本和路径要求严格，不如前两种稳定。
|
||||||
|
|
||||||
|
> 注意：分批 tar 会临时占用额外磁盘空间，加载完成后可删除 tar 包。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
58
scripts/Dockerfile.bigcodebench-sandbox
Normal file
58
scripts/Dockerfile.bigcodebench-sandbox
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
# bigcodebench-sandbox
|
||||||
|
# Minimal sandbox for running BigCodeBench/HumanEval code inside evalscope.
|
||||||
|
# Based on https://github.com/bigcode-project/bigcodebench/blob/main/Docker/Evaluate.Dockerfile
|
||||||
|
# but stripped to only the runtime dependencies needed to execute benchmark tests.
|
||||||
|
# Uses python:3.9-slim because some pinned packages (e.g. numba==0.55.0) do not have
|
||||||
|
# wheels for newer Python versions.
|
||||||
|
# Uses Chinese mirrors for apt/pip to speed up builds inside mainland network.
|
||||||
|
|
||||||
|
FROM python:3.9-slim
|
||||||
|
|
||||||
|
# Use Chinese Debian mirror
|
||||||
|
RUN sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
sed -i 's|http://deb.debian.org/debian-security|http://mirrors.tuna.tsinghua.edu.cn/debian-security|g' /etc/apt/sources.list.d/debian.sources && \
|
||||||
|
sed -i 's|http://deb.debian.org/debian|http://mirrors.tuna.tsinghua.edu.cn/debian|g' /etc/apt/sources.list || true
|
||||||
|
|
||||||
|
# Configure Chinese PyPI mirror
|
||||||
|
RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \
|
||||||
|
pip config set global.timeout 2000
|
||||||
|
|
||||||
|
# Install system dependencies needed by the scientific/Python packages
|
||||||
|
RUN apt-get update && apt-get install -y \
|
||||||
|
git \
|
||||||
|
g++ \
|
||||||
|
python3-tk \
|
||||||
|
zip \
|
||||||
|
unzip \
|
||||||
|
procps \
|
||||||
|
r-base \
|
||||||
|
libgdal-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libpng-dev \
|
||||||
|
pkg-config \
|
||||||
|
python3-dev \
|
||||||
|
python3-matplotlib \
|
||||||
|
libgl1 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Upgrade pip
|
||||||
|
RUN pip install --upgrade pip
|
||||||
|
|
||||||
|
# Add a non-root user (matches upstream image conventions)
|
||||||
|
RUN adduser --disabled-password --gecos "" bigcodebenchuser
|
||||||
|
|
||||||
|
# Copy and install the BigCodeBench evaluation requirements.
|
||||||
|
# These are the 70+ Python libraries referenced by the benchmark test cases.
|
||||||
|
COPY requirements-eval.txt /tmp/requirements-eval.txt
|
||||||
|
RUN pip install -I --timeout 2000 -r /tmp/requirements-eval.txt
|
||||||
|
|
||||||
|
# Ensure a compatible datasets version
|
||||||
|
RUN pip install datasets==2.17.0
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
RUN chown -R bigcodebenchuser:bigcodebenchuser /app && \
|
||||||
|
chmod -R 777 /app
|
||||||
|
|
||||||
|
# For evalscope sandbox: keep container alive
|
||||||
|
ENTRYPOINT []
|
||||||
|
CMD ["tail", "-f", "/dev/null"]
|
||||||
292
scripts/deploy_evalscope.sh
Executable file
292
scripts/deploy_evalscope.sh
Executable file
@ -0,0 +1,292 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ============================================================
|
||||||
|
# EvalScope 完整部署脚本
|
||||||
|
# 用法: bash deploy_evalscope.sh <BASE_DIR> [TOKEN]
|
||||||
|
# 示例: bash deploy_evalscope.sh /data1/sora ms-3d554a39-6e07-496d-8022-0b0ee64a6389
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 1. 参数解析
|
||||||
|
# --------------------------------------------------
|
||||||
|
BASE_DIR="${1:-/data1/sora}" # 基础目录，默认 /data1/sora
|
||||||
|
DEFAULT_TOKEN="ms-3d554a39-6e07-496d-8022-0b0ee64a6389"
|
||||||
|
MODELSCOPE_TOKEN="${2:-$DEFAULT_TOKEN}"
|
||||||
|
|
||||||
|
# 派生路径
|
||||||
|
EVALSCOPE_DIR="$BASE_DIR/evalscope"
|
||||||
|
DOCKER_DIR="$EVALSCOPE_DIR/docker"
|
||||||
|
IMAGES_DIR="$DOCKER_DIR/images"
|
||||||
|
CONTAINERD_DIR="$DOCKER_DIR/containerd"
|
||||||
|
SWE_IMAGES_DIR="$DOCKER_DIR/swe_images"
|
||||||
|
DATASETS_DIR="$EVALSCOPE_DIR/datasets"
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 2. 安装依赖
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 安装 modelscope..."
|
||||||
|
pip install modelscope
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 3. 登录 ModelScope
|
||||||
|
# --------------------------------------------------
|
||||||
|
if [ -n "$MODELSCOPE_TOKEN" ]; then
|
||||||
|
echo "==> 登录 ModelScope..."
|
||||||
|
modelscope login --token "$MODELSCOPE_TOKEN"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 4. 下载数据集
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 下载 evalscope 数据集..."
|
||||||
|
mkdir -p "$DATASETS_DIR"
|
||||||
|
python3 -c "
|
||||||
|
from modelscope.hub.snapshot_download import snapshot_download
|
||||||
|
snapshot_download(
|
||||||
|
'SoraAmami/evalscope-datasets',
|
||||||
|
repo_type='dataset',
|
||||||
|
cache_dir='$EVALSCOPE_DIR',
|
||||||
|
local_dir='$DATASETS_DIR'
|
||||||
|
)
|
||||||
|
"
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 5. 下载 Docker 镜像包
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 下载 evalscope Docker 镜像..."
|
||||||
|
mkdir -p "$DOCKER_DIR"
|
||||||
|
python3 -c "
|
||||||
|
from modelscope.hub.file_download import model_file_download
|
||||||
|
model_file_download(
|
||||||
|
model_id='SoraAmami/evalscope-docker',
|
||||||
|
file_path='evalscope-complete-py312.tar.gz',
|
||||||
|
local_dir='$DOCKER_DIR'
|
||||||
|
)
|
||||||
|
"
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 6. 配置 Docker 和 containerd 数据目录
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 配置 Docker 和 containerd..."
|
||||||
|
|
||||||
|
# 停止服务
|
||||||
|
sudo systemctl stop docker.socket 2>/dev/null || true
|
||||||
|
sudo systemctl stop docker 2>/dev/null || true
|
||||||
|
sudo systemctl stop containerd 2>/dev/null || true
|
||||||
|
|
||||||
|
# 创建数据目录
|
||||||
|
mkdir -p "$IMAGES_DIR"
|
||||||
|
mkdir -p "$CONTAINERD_DIR"
|
||||||
|
|
||||||
|
# 配置 Docker
|
||||||
|
echo "==> 写入 Docker 配置..."
|
||||||
|
sudo tee /etc/docker/daemon.json <<EOF
|
||||||
|
{
|
||||||
|
"data-root": "$IMAGES_DIR",
|
||||||
|
"features": {
|
||||||
|
"containerd-snapshotter": true
|
||||||
|
},
|
||||||
|
"registry-mirrors": [
|
||||||
|
"https://docker.m.daocloud.io",
|
||||||
|
"https://docker.1ms.run",
|
||||||
|
"https://hub.rat.dev",
|
||||||
|
"https://docker.1panel.live",
|
||||||
|
"https://dockerproxy.com",
|
||||||
|
"https://hub-mirror.c.163.com",
|
||||||
|
"https://mirror.baidubce.com",
|
||||||
|
"https://docker.mirrors.ustc.edu.cn",
|
||||||
|
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||||
|
"https://docker.nju.edu.cn",
|
||||||
|
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 配置 containerd
|
||||||
|
echo "==> 写入 containerd 配置..."
|
||||||
|
sudo tee /etc/containerd/config.toml <<EOF
|
||||||
|
root = "$CONTAINERD_DIR"
|
||||||
|
state = "/run/containerd"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# 重启服务
|
||||||
|
sudo systemctl reset-failed docker.service 2>/dev/null || true
|
||||||
|
sudo systemctl start containerd
|
||||||
|
sudo systemctl start docker
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 7. 加载 evalscope 环境镜像
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 加载 evalscope Docker 镜像..."
|
||||||
|
docker load -i "$DOCKER_DIR/evalscope-complete-py312.tar.gz"
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 8. 克隆代码仓库
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo "==> 克隆 evalscope 代码..."
|
||||||
|
cd "$EVALSCOPE_DIR"
|
||||||
|
if [ ! -d "evalstone" ]; then
|
||||||
|
git clone https://git.meta-stone.net/sora/evalstone.git
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 9. 准备 sandbox 镜像
|
||||||
|
# --------------------------------------------------
|
||||||
|
SANDBOX_IMAGES_DIR="$DOCKER_DIR/sandbox_images"
|
||||||
|
mkdir -p "$SANDBOX_IMAGES_DIR"
|
||||||
|
|
||||||
|
# 函数：带超时拉取镜像，本地已有则跳过
|
||||||
|
pull_with_timeout() {
|
||||||
|
local image="$1"
|
||||||
|
local timeout_sec="${2:-600}"
|
||||||
|
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$image"; then
|
||||||
|
echo "镜像 $image 已存在，跳过 pull"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "拉取 $image (超时 ${timeout_sec}s)..."
|
||||||
|
if timeout "$timeout_sec" docker pull "$image"; then
|
||||||
|
echo "$image 拉取成功"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
echo "警告：$image 拉取失败或超时"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# 优先从 ModelScope 下载预打包的 sandbox 镜像，避免 DockerHub 拉取慢/失败
|
||||||
|
download_sandbox_image_from_modelscope() {
|
||||||
|
local file="$1"
|
||||||
|
if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then
|
||||||
|
echo "sandbox 镜像包 $file 已存在，跳过下载"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "==> 从 ModelScope 下载 $file ..."
|
||||||
|
# 优先使用新版 modelscope_hub API（支持 dataset repo）
|
||||||
|
python3 -c "
|
||||||
|
from modelscope_hub import HubApi
|
||||||
|
api = HubApi()
|
||||||
|
api.download_file(
|
||||||
|
repo_id='SoraAmami/evalscope-sandbox-images',
|
||||||
|
repo_type='dataset',
|
||||||
|
path='$file',
|
||||||
|
local_dir='$SANDBOX_IMAGES_DIR'
|
||||||
|
)
|
||||||
|
" && return 0
|
||||||
|
echo "警告：从 ModelScope 下载 $file 失败"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
load_sandbox_image() {
|
||||||
|
local file="$1"
|
||||||
|
local expected_image="$2"
|
||||||
|
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "$expected_image"; then
|
||||||
|
echo "镜像 $expected_image 已存在，跳过加载"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then
|
||||||
|
echo "==> 加载 $file ..."
|
||||||
|
docker load -i "$SANDBOX_IMAGES_DIR/$file"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# 9.1 bigcodebench-sandbox:latest
|
||||||
|
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then
|
||||||
|
if ! load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest"; then
|
||||||
|
download_sandbox_image_from_modelscope "bigcodebench-sandbox.tar.gz" && \
|
||||||
|
load_sandbox_image "bigcodebench-sandbox.tar.gz" "bigcodebench-sandbox:latest"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 9.2 python:3.11-slim
|
||||||
|
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then
|
||||||
|
if ! load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim"; then
|
||||||
|
download_sandbox_image_from_modelscope "python-3.11-slim.tar.gz" && \
|
||||||
|
load_sandbox_image "python-3.11-slim.tar.gz" "python:3.11-slim"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 9.3 fallback：如果 ModelScope 失败，尝试从 DockerHub / 镜像站拉取
|
||||||
|
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench-sandbox:latest"; then
|
||||||
|
echo "==> 尝试 fallback：拉取 bigcodebench 基础镜像并构建 sandbox..."
|
||||||
|
pull_with_timeout "bigcodebench/bigcodebench-evaluate:latest" 600 || true
|
||||||
|
if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "bigcodebench/bigcodebench-evaluate:latest"; then
|
||||||
|
echo "==> 构建 bigcodebench-sandbox..."
|
||||||
|
mkdir -p /tmp/bigcodebench-sandbox-ctx
|
||||||
|
docker build -t bigcodebench-sandbox:latest -f - /tmp/bigcodebench-sandbox-ctx <<'EOF' || echo "警告：bigcodebench-sandbox 构建失败"
|
||||||
|
FROM bigcodebench/bigcodebench-evaluate:latest
|
||||||
|
ENTRYPOINT []
|
||||||
|
CMD ["tail", "-f", "/dev/null"]
|
||||||
|
EOF
|
||||||
|
else
|
||||||
|
echo "警告：bigcodebench 基础镜像不存在，跳过 bigcodebench-sandbox 构建"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "python:3.11-slim"; then
|
||||||
|
pull_with_timeout "python:3.11-slim" 300 || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 10. 下载 SWE-Bench 镜像包（可选）
|
||||||
|
# --------------------------------------------------
|
||||||
|
if [ -n "$MODELSCOPE_TOKEN" ]; then
|
||||||
|
echo "==> 下载 SWE-Bench 镜像包..."
|
||||||
|
mkdir -p "$SWE_IMAGES_DIR"
|
||||||
|
modelscope download \
|
||||||
|
--repo-type dataset \
|
||||||
|
--local_dir "$SWE_IMAGES_DIR" \
|
||||||
|
SoraAmami/swe-bench-verified-images || echo "警告：SWE-Bench 镜像下载失败"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 11. 加载 SWE-Bench 镜像（如果存在）
|
||||||
|
# --------------------------------------------------
|
||||||
|
if [ -d "$SWE_IMAGES_DIR" ] && ls "$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz 1>/dev/null 2>&1; then
|
||||||
|
echo "==> 加载 SWE-Bench 镜像..."
|
||||||
|
TARS=("$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz)
|
||||||
|
TOTAL=${#TARS[@]}
|
||||||
|
echo "共找到 $TOTAL 个镜像包"
|
||||||
|
|
||||||
|
IDX=0
|
||||||
|
for TAR in "${TARS[@]}"; do
|
||||||
|
IDX=$((IDX + 1))
|
||||||
|
echo "[$IDX/$TOTAL] 加载 $(basename "$TAR")..."
|
||||||
|
docker load -i "$TAR"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "SWE-Bench 镜像加载完成，已加载 $(docker images | grep '^swebench/' | wc -l) 个"
|
||||||
|
else
|
||||||
|
echo "跳过 SWE-Bench 镜像加载（未找到镜像包）"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 12. 验证部署
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "========================================"
|
||||||
|
echo "部署完成！"
|
||||||
|
echo "========================================"
|
||||||
|
echo "基础目录: $BASE_DIR"
|
||||||
|
echo "EvalScope: $EVALSCOPE_DIR"
|
||||||
|
echo "数据集: $DATASETS_DIR"
|
||||||
|
echo "Docker 数据: $IMAGES_DIR"
|
||||||
|
echo "Containerd: $CONTAINERD_DIR"
|
||||||
|
echo ""
|
||||||
|
echo "Docker Root Dir:"
|
||||||
|
docker info 2>/dev/null | grep "Docker Root Dir" || echo "Docker 未运行"
|
||||||
|
echo ""
|
||||||
|
echo "已加载镜像:"
|
||||||
|
docker images | grep -E "evalscope|swebench|bigcodebench|python" || true
|
||||||
|
echo ""
|
||||||
|
echo "运行 EvalScope:"
|
||||||
|
echo " cd $EVALSCOPE_DIR"
|
||||||
|
echo " docker run -it --rm \\"
|
||||||
|
echo " --network host \\"
|
||||||
|
echo " -v $EVALSCOPE_DIR:/opt/evalscope \\"
|
||||||
|
echo " -v /var/run/docker.sock:/var/run/docker.sock \\"
|
||||||
|
echo " evalscope-complete-py312:latest \\"
|
||||||
|
echo " bash"
|
||||||
|
echo "========================================"
|
||||||
Loading…
x
Reference in New Issue
Block a user