From 5532c9403e37cfed38a1c729faa7618b2206f55f Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 27 Jul 2026 06:22:39 +0000 Subject: [PATCH] Add checkpoint skip for loaded images and --download-swe flag --- README.md | 14 +++++- scripts/deploy_evalscope.sh | 92 ++++++++++++++++++++++++++++++++++--- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5efa558..3ab3027 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,19 @@ python bash/run.py \ ## 3. Docker 镜像下载与部署 -根据是否跑 SWE-bench，选择下面两种方式之一。 +最简方式：直接运行一键部署脚本。 + +```bash +# 默认不下载 SWE-Bench 镜像（体积大） +bash scripts/deploy_evalscope.sh /data1/sora + +# 如果需要跑 SWE-Bench，加 --download-swe +bash scripts/deploy_evalscope.sh --download-swe /data1/sora +``` + +脚本会自动完成：安装 modelscope、登录、下载数据集、下载/加载 Docker 镜像、配置 Docker/containerd、加载 sandbox 镜像。重复执行时会通过 checkpoint 自动跳过已加载的镜像。 + +如果需要手动分步操作，参考下面两种方式。 ### 3.1 方式 A：不测 SWE-bench 的 Docker 配置 diff --git a/scripts/deploy_evalscope.sh b/scripts/deploy_evalscope.sh index fc51b12..b922c8b 100755 --- a/scripts/deploy_evalscope.sh +++ b/scripts/deploy_evalscope.sh @@ -1,8 +1,13 @@ #!/bin/bash # ============================================================ # EvalScope 完整部署脚本 -# 用法: bash deploy_evalscope.sh [TOKEN] -# 示例: bash deploy_evalscope.sh /data1/sora ms-3d554a39-6e07-496d-8022-0b0ee64a6389 +# 用法: bash deploy_evalscope.sh [OPTIONS] [TOKEN] +# 示例: +# bash deploy_evalscope.sh /data1/sora +# bash deploy_evalscope.sh --download-swe /data1/sora +# +# 断点续传：已加载的镜像/tar 包会记录在 /evalscope/docker/.deploy_checkpoint +# 重复执行时会自动跳过，避免二次加载。 # ============================================================ set -euo pipefail @@ -10,6 +15,36 @@ set -euo pipefail # -------------------------------------------------- # 1. 参数解析 # -------------------------------------------------- +DOWNLOAD_SWE=0 +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --download-swe) + DOWNLOAD_SWE=1 + shift + ;; + --help|-h) + echo "用法: bash deploy_evalscope.sh [OPTIONS] [TOKEN]" + echo "选项:" + echo " --download-swe 同时下载 SWE-Bench 镜像包（默认不下载，体积很大）" + echo " -h, --help 显示帮助" + exit 0 + ;; + -*) + echo "未知选项: $1" + echo "使用 --help 查看用法" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + BASE_DIR="${1:-/data1/sora}" # 基础目录，默认 /data1/sora DEFAULT_TOKEN="ms-3d554a39-6e07-496d-8022-0b0ee64a6389" MODELSCOPE_TOKEN="${2:-$DEFAULT_TOKEN}" @@ -21,6 +56,25 @@ IMAGES_DIR="$DOCKER_DIR/images" CONTAINERD_DIR="$DOCKER_DIR/containerd" SWE_IMAGES_DIR="$DOCKER_DIR/swe_images" DATASETS_DIR="$EVALSCOPE_DIR/datasets" +CHECKPOINT_FILE="$DOCKER_DIR/.deploy_checkpoint" + +# 断点：记录/检查已加载的镜像或 tar 包 +mark_loaded() { + local key="$1" + mkdir -p "$DOCKER_DIR" + touch "$CHECKPOINT_FILE" + if ! grep -qx "$key" "$CHECKPOINT_FILE" 2>/dev/null; then + echo "$key" >> "$CHECKPOINT_FILE" + fi +} + +is_loaded() { + local key="$1" + if [ -f "$CHECKPOINT_FILE" ] && grep -qx "$key" "$CHECKPOINT_FILE" 2>/dev/null; then + return 0 + fi + return 1 +} # -------------------------------------------------- # 2. 安装依赖 @@ -119,7 +173,14 @@ sudo systemctl start docker # 7. 加载 evalscope 环境镜像 # -------------------------------------------------- echo "==> 加载 evalscope Docker 镜像..." -docker load -i "$DOCKER_DIR/evalscope-complete-py312.tar.gz" +if docker images --format '{{.Repository}}:{{.Tag}}' | grep -qx "evalscope-complete-py312:latest"; then + echo "evalscope-complete-py312:latest 已存在，跳过加载" +elif is_loaded "evalscope-complete-py312.tar.gz"; then + echo "evalscope-complete-py312.tar.gz 已加载过（checkpoint），跳过加载" +else + docker load -i "$DOCKER_DIR/evalscope-complete-py312.tar.gz" + mark_loaded "evalscope-complete-py312.tar.gz" +fi # -------------------------------------------------- # 8. 克隆代码仓库 @@ -167,9 +228,14 @@ load_sandbox_image() { echo "镜像 $expected_image 已存在，跳过加载" return 0 fi + if is_loaded "$file"; then + echo "镜像包 $file 已加载过（checkpoint），跳过加载" + return 0 + fi if [ -f "$SANDBOX_IMAGES_DIR/$file" ]; then echo "==> 加载 $file ..." docker load -i "$SANDBOX_IMAGES_DIR/$file" + mark_loaded "$file" return 0 fi return 1 @@ -196,15 +262,17 @@ else fi # -------------------------------------------------- -# 10. 下载 SWE-Bench 镜像包（可选） +# 10. 下载 SWE-Bench 镜像包（默认不下载，需加 --download-swe） # -------------------------------------------------- -if [ -n "$MODELSCOPE_TOKEN" ]; then +if [ "$DOWNLOAD_SWE" -eq 1 ]; then echo "==> 下载 SWE-Bench 镜像包..." mkdir -p "$SWE_IMAGES_DIR" modelscope download \ --repo-type dataset \ --local_dir "$SWE_IMAGES_DIR" \ SoraAmami/swe-bench-verified-images || echo "警告：SWE-Bench 镜像下载失败" +else + echo "==> 跳过 SWE-Bench 镜像包下载（如需下载请加 --download-swe）" fi # -------------------------------------------------- @@ -217,13 +285,23 @@ if [ -d "$SWE_IMAGES_DIR" ] && ls "$SWE_IMAGES_DIR"/swebench_batch_*.tar.gz 1>/d echo "共找到 $TOTAL 个镜像包" IDX=0 + LOADED_COUNT=0 + SKIPPED_COUNT=0 for TAR in "${TARS[@]}"; do IDX=$((IDX + 1)) - echo "[$IDX/$TOTAL] 加载 $(basename "$TAR")..." + BASENAME=$(basename "$TAR") + if is_loaded "$BASENAME"; then + echo "[$IDX/$TOTAL] 跳过（已加载过）: $BASENAME" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + echo "[$IDX/$TOTAL] 加载 $BASENAME ..." docker load -i "$TAR" + mark_loaded "$BASENAME" + LOADED_COUNT=$((LOADED_COUNT + 1)) done - echo "SWE-Bench 镜像加载完成，已加载 $(docker images | grep '^swebench/' | wc -l) 个" + echo "SWE-Bench 镜像加载完成：本次加载 $LOADED_COUNT 个，跳过 $SKIPPED_COUNT 个，已加载 $(docker images | grep -c '^swebench/' || true) 个" else echo "跳过 SWE-Bench 镜像加载（未找到镜像包）" fi