diff --git a/bash/load_swe_images.sh b/bash/load_swe_images.sh new file mode 100755 index 0000000..6c63844 --- /dev/null +++ b/bash/load_swe_images.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# 从 save_swe_images.sh 生成的 tar.gz 批量加载 swebench 实例镜像。 + +set -e + +INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}" + +if [ ! -d "$INPUT_DIR" ]; then + echo "错误:目录不存在 $INPUT_DIR" + exit 1 +fi + +TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz) +TOTAL=${#TARS[@]} + +echo "共找到 $TOTAL 个镜像包,开始加载..." + +IDX=0 +for TAR in "${TARS[@]}"; do + IDX=$((IDX + 1)) + echo "[$IDX/$TOTAL] 加载 $TAR" + docker load -i "$TAR" +done + +echo "完成。已加载镜像数:" +docker images | grep '^swebench/' | wc -l diff --git a/bash/save_swe_images.sh b/bash/save_swe_images.sh new file mode 100755 index 0000000..ed4cc65 --- /dev/null +++ b/bash/save_swe_images.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# 将本地已拉取的 swebench 实例镜像打包保存,方便通过移动硬盘/内网传送到其他机器。 +# 注意:swe_bench_verified 全量约 500 个镜像、~2TB,无法上传到 ModelScope 这类代码/文件仓库。 + +set -e + +OUTPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}" +BATCH_SIZE="${2:-50}" + +mkdir -p "$OUTPUT_DIR" + +echo "正在列出所有 swebench 镜像..." +mapfile -t IMAGES < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep '^swebench/') +TOTAL=${#IMAGES[@]} + +echo "共找到 $TOTAL 个 swebench 镜像,将按每批 $BATCH_SIZE 个保存到 $OUTPUT_DIR" + +BATCH=() +BATCH_IDX=0 +COUNT=0 + +for IMG in "${IMAGES[@]}"; do + BATCH+=("$IMG") + COUNT=$((COUNT + 1)) + + if ((${#BATCH[@]} >= BATCH_SIZE)) || ((COUNT == TOTAL)); then + BATCH_IDX=$((BATCH_IDX + 1)) + TAR_FILE="$OUTPUT_DIR/swebench_batch_$(printf '%03d' $BATCH_IDX).tar" + echo "[$COUNT/$TOTAL] 保存批次 $BATCH_IDX -> $TAR_FILE" + docker save "${BATCH[@]}" -o "$TAR_FILE" + gzip -f "$TAR_FILE" + BATCH=() + fi +done + +echo "完成。镜像列表:" +ls -lh "$OUTPUT_DIR" diff --git a/myread.md b/myread.md index 260996b..64b4b0b 100644 --- a/myread.md +++ b/myread.md @@ -165,45 +165,44 @@ SWE-bench 镜像数量多、体积大,建议把 Docker `data-root` 和 contain ```bash # 1. 停止 Docker -systemctl stop docker.socket -systemctl stop docker -systemctl stop containerd +sudo systemctl stop docker.socket +sudo systemctl stop docker +sudo systemctl stop containerd # 2. 创建新的数据目录 mkdir -p /data1/sora/evalscope/docker/images mkdir -p /data1/sora/evalscope/docker/containerd # 3. 配置国内镜像加速(多填几个,自动轮询) -cat > /etc/docker/daemon.json <<'EOF' +sudo tee /etc/docker/daemon.json <<-'EOF' { "data-root": "/data1/sora/evalscope/docker/images", "registry-mirrors": [ "https://docker.m.daocloud.io", "https://docker.1ms.run", - "https://docker.1panel.live", "https://hub.rat.dev", + "https://docker.1panel.live", + "https://dockerproxy.com", "https://hub-mirror.c.163.com", "https://mirror.baidubce.com", - "https://dockerproxy.com", - "https://docker.mirrors.aliyun.com", + "https://docker.mirrors.ustc.edu.cn", "https://docker.mirrors.sjtug.sjtu.edu.cn", "https://docker.nju.edu.cn", - "https://docker.mirrors.ustc.edu.cn", "https://docker.mirrors.tuna.tsinghua.edu.cn" ] } EOF # 4. 配置 containerd 数据目录 -cat > /etc/containerd/config.toml <<'EOF' +sudo tee /etc/containerd/config.toml <<-'EOF' root = "/data1/sora/evalscope/docker/containerd" state = "/run/containerd" EOF # 5. 重启 -systemctl reset-failed docker.service -systemctl start containerd -systemctl start docker +sudo systemctl reset-failed docker.service +sudo systemctl start containerd +sudo systemctl start docker ``` #### 2.4.2 拉取/构建代码执行镜像 @@ -268,6 +267,26 @@ python bash/pull_swe_bench_images.py \ > 2. 降低 `--max-workers` 到 1; > 3. 使用 `tmux` 挂后台运行,避免 SSH 断连导致中断。 +#### 2.4.4 SWE-bench 镜像的本地备份与迁移(可选) + +SWE-bench 全量镜像约 500 个、总大小约 **2TB**,无法上传到 ModelScope 这类代码/文件仓库分发。如需把已拉好的镜像迁移到另一台机器,建议用移动硬盘或内网直接拷贝 Docker `data-root`,或使用下面脚本分批导出/导入: + +**源机器:打包** + +```bash +bash bash/save_swe_images.sh /data1/sora/evalscope/docker/swe_images 50 +``` + +脚本会把镜像按每 50 个一批保存为 `swebench_batch_001.tar.gz` 等。 + +**目标机器:加载** + +```bash +bash bash/load_swe_images.sh /data1/sora/evalscope/docker/swe_images +``` + +> 注意:分批 tar 会占用额外磁盘空间(约 2TB),请确保目标盘容量足够。 + --- ## 3. 环境准备