Add SWE image save/load scripts and update Docker config docs

This commit is contained in:
sora 2026-07-24 06:27:49 +00:00
parent 92f9c970e3
commit 10eecf787d
3 changed files with 94 additions and 12 deletions

26
bash/load_swe_images.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
# 从 save_swe_images.sh 生成的 tar.gz 批量加载 swebench 实例镜像。
set -e
INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
if [ ! -d "$INPUT_DIR" ]; then
echo "错误:目录不存在 $INPUT_DIR"
exit 1
fi
TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz)
TOTAL=${#TARS[@]}
echo "共找到 $TOTAL 个镜像包,开始加载..."
IDX=0
for TAR in "${TARS[@]}"; do
IDX=$((IDX + 1))
echo "[$IDX/$TOTAL] 加载 $TAR"
docker load -i "$TAR"
done
echo "完成。已加载镜像数:"
docker images | grep '^swebench/' | wc -l

37
bash/save_swe_images.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# 将本地已拉取的 swebench 实例镜像打包保存,方便通过移动硬盘/内网传送到其他机器。
# 注意swe_bench_verified 全量约 500 个镜像、~2TB无法上传到 ModelScope 这类代码/文件仓库。
set -e
OUTPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
BATCH_SIZE="${2:-50}"
mkdir -p "$OUTPUT_DIR"
echo "正在列出所有 swebench 镜像..."
mapfile -t IMAGES < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^swebench/')
TOTAL=${#IMAGES[@]}
echo "共找到 $TOTAL 个 swebench 镜像,将按每批 $BATCH_SIZE 个保存到 $OUTPUT_DIR"
BATCH=()
BATCH_IDX=0
COUNT=0
for IMG in "${IMAGES[@]}"; do
BATCH+=("$IMG")
COUNT=$((COUNT + 1))
if ((${#BATCH[@]} >= BATCH_SIZE)) || ((COUNT == TOTAL)); then
BATCH_IDX=$((BATCH_IDX + 1))
TAR_FILE="$OUTPUT_DIR/swebench_batch_$(printf '%03d' $BATCH_IDX).tar"
echo "[$COUNT/$TOTAL] 保存批次 $BATCH_IDX -> $TAR_FILE"
docker save "${BATCH[@]}" -o "$TAR_FILE"
gzip -f "$TAR_FILE"
BATCH=()
fi
done
echo "完成。镜像列表:"
ls -lh "$OUTPUT_DIR"

View File

@ -165,45 +165,44 @@ SWE-bench 镜像数量多、体积大,建议把 Docker `data-root` 和 contain
```bash
# 1. 停止 Docker
systemctl stop docker.socket
systemctl stop docker
systemctl stop containerd
sudo systemctl stop docker.socket
sudo systemctl stop docker
sudo systemctl stop containerd
# 2. 创建新的数据目录
mkdir -p /data1/sora/evalscope/docker/images
mkdir -p /data1/sora/evalscope/docker/containerd
# 3. 配置国内镜像加速(多填几个,自动轮询)
cat > /etc/docker/daemon.json <<'EOF'
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"data-root": "/data1/sora/evalscope/docker/images",
"registry-mirrors": [
"https://docker.m.daocloud.io",
"https://docker.1ms.run",
"https://docker.1panel.live",
"https://hub.rat.dev",
"https://docker.1panel.live",
"https://dockerproxy.com",
"https://hub-mirror.c.163.com",
"https://mirror.baidubce.com",
"https://dockerproxy.com",
"https://docker.mirrors.aliyun.com",
"https://docker.mirrors.ustc.edu.cn",
"https://docker.mirrors.sjtug.sjtu.edu.cn",
"https://docker.nju.edu.cn",
"https://docker.mirrors.ustc.edu.cn",
"https://docker.mirrors.tuna.tsinghua.edu.cn"
]
}
EOF
# 4. 配置 containerd 数据目录
cat > /etc/containerd/config.toml <<'EOF'
sudo tee /etc/containerd/config.toml <<-'EOF'
root = "/data1/sora/evalscope/docker/containerd"
state = "/run/containerd"
EOF
# 5. 重启
systemctl reset-failed docker.service
systemctl start containerd
systemctl start docker
sudo systemctl reset-failed docker.service
sudo systemctl start containerd
sudo systemctl start docker
```
#### 2.4.2 拉取/构建代码执行镜像
@ -268,6 +267,26 @@ python bash/pull_swe_bench_images.py \
> 2. 降低 `--max-workers` 到 1
> 3. 使用 `tmux` 挂后台运行,避免 SSH 断连导致中断。
#### 2.4.4 SWE-bench 镜像的本地备份与迁移(可选)
SWE-bench 全量镜像约 500 个、总大小约 **2TB**,无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器,建议用移动硬盘或内网直接拷贝 Docker `data-root`,或使用下面脚本分批导出/导入:
**源机器:打包**
```bash
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
```
脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。
**目标机器:加载**
```bash
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
```
> 注意:分批 tar 会占用额外磁盘空间(约 2TB请确保目标盘容量足够。
---
## 3. 环境准备