Add SWE image save/load scripts and update Docker config docs
This commit is contained in:
parent
92f9c970e3
commit
10eecf787d
26
bash/load_swe_images.sh
Executable file
26
bash/load_swe_images.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
# 从 save_swe_images.sh 生成的 tar.gz 批量加载 swebench 实例镜像。
|
||||
|
||||
set -e
|
||||
|
||||
INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
|
||||
|
||||
if [ ! -d "$INPUT_DIR" ]; then
|
||||
echo "错误:目录不存在 $INPUT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz)
|
||||
TOTAL=${#TARS[@]}
|
||||
|
||||
echo "共找到 $TOTAL 个镜像包,开始加载..."
|
||||
|
||||
IDX=0
|
||||
for TAR in "${TARS[@]}"; do
|
||||
IDX=$((IDX + 1))
|
||||
echo "[$IDX/$TOTAL] 加载 $TAR"
|
||||
docker load -i "$TAR"
|
||||
done
|
||||
|
||||
echo "完成。已加载镜像数:"
|
||||
docker images | grep '^swebench/' | wc -l
|
||||
37
bash/save_swe_images.sh
Executable file
37
bash/save_swe_images.sh
Executable file
@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# 将本地已拉取的 swebench 实例镜像打包保存,方便通过移动硬盘/内网传送到其他机器。
|
||||
# 注意:swe_bench_verified 全量约 500 个镜像、~2TB,无法上传到 ModelScope 这类代码/文件仓库。
|
||||
|
||||
set -e
|
||||
|
||||
OUTPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
|
||||
BATCH_SIZE="${2:-50}"
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
echo "正在列出所有 swebench 镜像..."
|
||||
mapfile -t IMAGES < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^swebench/')
|
||||
TOTAL=${#IMAGES[@]}
|
||||
|
||||
echo "共找到 $TOTAL 个 swebench 镜像,将按每批 $BATCH_SIZE 个保存到 $OUTPUT_DIR"
|
||||
|
||||
BATCH=()
|
||||
BATCH_IDX=0
|
||||
COUNT=0
|
||||
|
||||
for IMG in "${IMAGES[@]}"; do
|
||||
BATCH+=("$IMG")
|
||||
COUNT=$((COUNT + 1))
|
||||
|
||||
if ((${#BATCH[@]} >= BATCH_SIZE)) || ((COUNT == TOTAL)); then
|
||||
BATCH_IDX=$((BATCH_IDX + 1))
|
||||
TAR_FILE="$OUTPUT_DIR/swebench_batch_$(printf '%03d' $BATCH_IDX).tar"
|
||||
echo "[$COUNT/$TOTAL] 保存批次 $BATCH_IDX -> $TAR_FILE"
|
||||
docker save "${BATCH[@]}" -o "$TAR_FILE"
|
||||
gzip -f "$TAR_FILE"
|
||||
BATCH=()
|
||||
fi
|
||||
done
|
||||
|
||||
echo "完成。镜像列表:"
|
||||
ls -lh "$OUTPUT_DIR"
|
||||
43
myread.md
43
myread.md
@ -165,45 +165,44 @@ SWE-bench 镜像数量多、体积大,建议把 Docker `data-root` 和 contain
|
||||
|
||||
```bash
|
||||
# 1. 停止 Docker
|
||||
systemctl stop docker.socket
|
||||
systemctl stop docker
|
||||
systemctl stop containerd
|
||||
sudo systemctl stop docker.socket
|
||||
sudo systemctl stop docker
|
||||
sudo systemctl stop containerd
|
||||
|
||||
# 2. 创建新的数据目录
|
||||
mkdir -p /data1/sora/evalscope/docker/images
|
||||
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||
|
||||
# 3. 配置国内镜像加速(多填几个,自动轮询)
|
||||
cat > /etc/docker/daemon.json <<'EOF'
|
||||
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||
{
|
||||
"data-root": "/data1/sora/evalscope/docker/images",
|
||||
"registry-mirrors": [
|
||||
"https://docker.m.daocloud.io",
|
||||
"https://docker.1ms.run",
|
||||
"https://docker.1panel.live",
|
||||
"https://hub.rat.dev",
|
||||
"https://docker.1panel.live",
|
||||
"https://dockerproxy.com",
|
||||
"https://hub-mirror.c.163.com",
|
||||
"https://mirror.baidubce.com",
|
||||
"https://dockerproxy.com",
|
||||
"https://docker.mirrors.aliyun.com",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||
"https://docker.nju.edu.cn",
|
||||
"https://docker.mirrors.ustc.edu.cn",
|
||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
# 4. 配置 containerd 数据目录
|
||||
cat > /etc/containerd/config.toml <<'EOF'
|
||||
sudo tee /etc/containerd/config.toml <<-'EOF'
|
||||
root = "/data1/sora/evalscope/docker/containerd"
|
||||
state = "/run/containerd"
|
||||
EOF
|
||||
|
||||
# 5. 重启
|
||||
systemctl reset-failed docker.service
|
||||
systemctl start containerd
|
||||
systemctl start docker
|
||||
sudo systemctl reset-failed docker.service
|
||||
sudo systemctl start containerd
|
||||
sudo systemctl start docker
|
||||
```
|
||||
|
||||
#### 2.4.2 拉取/构建代码执行镜像
|
||||
@ -268,6 +267,26 @@ python bash/pull_swe_bench_images.py \
|
||||
> 2. 降低 `--max-workers` 到 1;
|
||||
> 3. 使用 `tmux` 挂后台运行,避免 SSH 断连导致中断。
|
||||
|
||||
#### 2.4.4 SWE-bench 镜像的本地备份与迁移(可选)
|
||||
|
||||
SWE-bench 全量镜像约 500 个、总大小约 **2TB**,无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器,建议用移动硬盘或内网直接拷贝 Docker `data-root`,或使用下面脚本分批导出/导入:
|
||||
|
||||
**源机器:打包**
|
||||
|
||||
```bash
|
||||
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
||||
```
|
||||
|
||||
脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。
|
||||
|
||||
**目标机器:加载**
|
||||
|
||||
```bash
|
||||
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||
```
|
||||
|
||||
> 注意:分批 tar 会占用额外磁盘空间(约 2TB),请确保目标盘容量足够。
|
||||
|
||||
---
|
||||
|
||||
## 3. 环境准备
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user