Add SWE image save/load scripts and update Docker config docs
This commit is contained in:
parent
92f9c970e3
commit
10eecf787d
26
bash/load_swe_images.sh
Executable file
26
bash/load_swe_images.sh
Executable file
@ -0,0 +1,26 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# 从 save_swe_images.sh 生成的 tar.gz 批量加载 swebench 实例镜像。
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
|
||||||
|
|
||||||
|
if [ ! -d "$INPUT_DIR" ]; then
|
||||||
|
echo "错误:目录不存在 $INPUT_DIR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz)
|
||||||
|
TOTAL=${#TARS[@]}
|
||||||
|
|
||||||
|
echo "共找到 $TOTAL 个镜像包,开始加载..."
|
||||||
|
|
||||||
|
IDX=0
|
||||||
|
for TAR in "${TARS[@]}"; do
|
||||||
|
IDX=$((IDX + 1))
|
||||||
|
echo "[$IDX/$TOTAL] 加载 $TAR"
|
||||||
|
docker load -i "$TAR"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "完成。已加载镜像数:"
|
||||||
|
docker images | grep '^swebench/' | wc -l
|
||||||
37
bash/save_swe_images.sh
Executable file
37
bash/save_swe_images.sh
Executable file
@ -0,0 +1,37 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# 将本地已拉取的 swebench 实例镜像打包保存,方便通过移动硬盘/内网传送到其他机器。
|
||||||
|
# 注意:swe_bench_verified 全量约 500 个镜像、~2TB,无法上传到 ModelScope 这类代码/文件仓库。
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
OUTPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
|
||||||
|
BATCH_SIZE="${2:-50}"
|
||||||
|
|
||||||
|
mkdir -p "$OUTPUT_DIR"
|
||||||
|
|
||||||
|
echo "正在列出所有 swebench 镜像..."
|
||||||
|
mapfile -t IMAGES < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^swebench/')
|
||||||
|
TOTAL=${#IMAGES[@]}
|
||||||
|
|
||||||
|
echo "共找到 $TOTAL 个 swebench 镜像,将按每批 $BATCH_SIZE 个保存到 $OUTPUT_DIR"
|
||||||
|
|
||||||
|
BATCH=()
|
||||||
|
BATCH_IDX=0
|
||||||
|
COUNT=0
|
||||||
|
|
||||||
|
for IMG in "${IMAGES[@]}"; do
|
||||||
|
BATCH+=("$IMG")
|
||||||
|
COUNT=$((COUNT + 1))
|
||||||
|
|
||||||
|
if ((${#BATCH[@]} >= BATCH_SIZE)) || ((COUNT == TOTAL)); then
|
||||||
|
BATCH_IDX=$((BATCH_IDX + 1))
|
||||||
|
TAR_FILE="$OUTPUT_DIR/swebench_batch_$(printf '%03d' $BATCH_IDX).tar"
|
||||||
|
echo "[$COUNT/$TOTAL] 保存批次 $BATCH_IDX -> $TAR_FILE"
|
||||||
|
docker save "${BATCH[@]}" -o "$TAR_FILE"
|
||||||
|
gzip -f "$TAR_FILE"
|
||||||
|
BATCH=()
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "完成。镜像列表:"
|
||||||
|
ls -lh "$OUTPUT_DIR"
|
||||||
43
myread.md
43
myread.md
@ -165,45 +165,44 @@ SWE-bench 镜像数量多、体积大,建议把 Docker `data-root` 和 contain
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 1. 停止 Docker
|
# 1. 停止 Docker
|
||||||
systemctl stop docker.socket
|
sudo systemctl stop docker.socket
|
||||||
systemctl stop docker
|
sudo systemctl stop docker
|
||||||
systemctl stop containerd
|
sudo systemctl stop containerd
|
||||||
|
|
||||||
# 2. 创建新的数据目录
|
# 2. 创建新的数据目录
|
||||||
mkdir -p /data1/sora/evalscope/docker/images
|
mkdir -p /data1/sora/evalscope/docker/images
|
||||||
mkdir -p /data1/sora/evalscope/docker/containerd
|
mkdir -p /data1/sora/evalscope/docker/containerd
|
||||||
|
|
||||||
# 3. 配置国内镜像加速(多填几个,自动轮询)
|
# 3. 配置国内镜像加速(多填几个,自动轮询)
|
||||||
cat > /etc/docker/daemon.json <<'EOF'
|
sudo tee /etc/docker/daemon.json <<-'EOF'
|
||||||
{
|
{
|
||||||
"data-root": "/data1/sora/evalscope/docker/images",
|
"data-root": "/data1/sora/evalscope/docker/images",
|
||||||
"registry-mirrors": [
|
"registry-mirrors": [
|
||||||
"https://docker.m.daocloud.io",
|
"https://docker.m.daocloud.io",
|
||||||
"https://docker.1ms.run",
|
"https://docker.1ms.run",
|
||||||
"https://docker.1panel.live",
|
|
||||||
"https://hub.rat.dev",
|
"https://hub.rat.dev",
|
||||||
|
"https://docker.1panel.live",
|
||||||
|
"https://dockerproxy.com",
|
||||||
"https://hub-mirror.c.163.com",
|
"https://hub-mirror.c.163.com",
|
||||||
"https://mirror.baidubce.com",
|
"https://mirror.baidubce.com",
|
||||||
"https://dockerproxy.com",
|
"https://docker.mirrors.ustc.edu.cn",
|
||||||
"https://docker.mirrors.aliyun.com",
|
|
||||||
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
"https://docker.mirrors.sjtug.sjtu.edu.cn",
|
||||||
"https://docker.nju.edu.cn",
|
"https://docker.nju.edu.cn",
|
||||||
"https://docker.mirrors.ustc.edu.cn",
|
|
||||||
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
"https://docker.mirrors.tuna.tsinghua.edu.cn"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# 4. 配置 containerd 数据目录
|
# 4. 配置 containerd 数据目录
|
||||||
cat > /etc/containerd/config.toml <<'EOF'
|
sudo tee /etc/containerd/config.toml <<-'EOF'
|
||||||
root = "/data1/sora/evalscope/docker/containerd"
|
root = "/data1/sora/evalscope/docker/containerd"
|
||||||
state = "/run/containerd"
|
state = "/run/containerd"
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
# 5. 重启
|
# 5. 重启
|
||||||
systemctl reset-failed docker.service
|
sudo systemctl reset-failed docker.service
|
||||||
systemctl start containerd
|
sudo systemctl start containerd
|
||||||
systemctl start docker
|
sudo systemctl start docker
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 2.4.2 拉取/构建代码执行镜像
|
#### 2.4.2 拉取/构建代码执行镜像
|
||||||
@ -268,6 +267,26 @@ python bash/pull_swe_bench_images.py \
|
|||||||
> 2. 降低 `--max-workers` 到 1;
|
> 2. 降低 `--max-workers` 到 1;
|
||||||
> 3. 使用 `tmux` 挂后台运行,避免 SSH 断连导致中断。
|
> 3. 使用 `tmux` 挂后台运行,避免 SSH 断连导致中断。
|
||||||
|
|
||||||
|
#### 2.4.4 SWE-bench 镜像的本地备份与迁移(可选)
|
||||||
|
|
||||||
|
SWE-bench 全量镜像约 500 个、总大小约 **2TB**,无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器,建议用移动硬盘或内网直接拷贝 Docker `data-root`,或使用下面脚本分批导出/导入:
|
||||||
|
|
||||||
|
**源机器:打包**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50
|
||||||
|
```
|
||||||
|
|
||||||
|
脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。
|
||||||
|
|
||||||
|
**目标机器:加载**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注意:分批 tar 会占用额外磁盘空间(约 2TB),请确保目标盘容量足够。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. 环境准备
|
## 3. 环境准备
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user