From a37a1165b2c1036d6d8f13605f7c89999145469b Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Wed, 2 Sep 2026 09:44:17 +0000 Subject: [PATCH] Preload Pier uv and mini-swe-agent wheels so DeepSWE sandbox builds skip GitHub. Co-authored-by: Cursor --- .gitignore | 3 + .../preload_deep_swe_agent_offline.sh | 174 ++++++++++++++ .../benchmarks/deep_swe/deep_swe_adapter.py | 4 + .../deep_swe/pier_offline_install.py | 214 ++++++++++++++++++ 4 files changed, 395 insertions(+) create mode 100755 bash/images_load/preload_deep_swe_agent_offline.sh create mode 100644 evalscope/evalscope/benchmarks/deep_swe/pier_offline_install.py diff --git a/.gitignore b/.gitignore index b3d105c..9708a1c 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ config_private.py node_modules/ *.zip *.tar.gz + +# DeepSWE Pier 离线 uv / mini-swe-agent wheel +/bash/images_load/offline_pier_agent/ diff --git a/bash/images_load/preload_deep_swe_agent_offline.sh b/bash/images_load/preload_deep_swe_agent_offline.sh new file mode 100755 index 0000000..a1c929d --- /dev/null +++ b/bash/images_load/preload_deep_swe_agent_offline.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# ============================================================ +# 预下载 Pier mini-swe-agent 安装依赖（uv 二进制 + wheel） +# +# DeepSWE 每个任务镜像都会再执行： +# curl https://astral.sh/uv/0.7.13/install.sh | sh +# uv tool install mini-swe-agent +# FROM 镜像不同，Docker 层缓存不共享，全样本会反复打 GitHub/PyPI。 +# +# 本脚本把 uv 和 mini-swe-agent 的 wheel 放到 +# bash/images_load/offline_pier_agent/ +# 评测时 evalscope 会 COPY 进构建上下文，构建不再访问 GitHub。 +# +# 用法： +# bash bash/images_load/preload_deep_swe_agent_offline.sh +# ============================================================ + +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +VENDOR_DIR="${PIER_OFFLINE_AGENT_DIR:-${PROJECT_ROOT}/bash/images_load/offline_pier_agent}" +WHEEL_DIR="${VENDOR_DIR}/wheels" +UV_BIN="${VENDOR_DIR}/uv" +HOST_UV="${UV:-}" +PIP_INDEXES=( + "https://pypi.tuna.tsinghua.edu.cn/simple" + "https://mirrors.aliyun.com/pypi/simple/" +) +UV_MIRRORS=( + "https://github.com/astral-sh/uv/releases/download/0.7.13/uv-x86_64-unknown-linux-gnu.tar.gz" + "https://ghproxy.net/https://github.com/astral-sh/uv/releases/download/0.7.13/uv-x86_64-unknown-linux-gnu.tar.gz" + "https://mirror.ghproxy.com/https://github.com/astral-sh/uv/releases/download/0.7.13/uv-x86_64-unknown-linux-gnu.tar.gz" +) +PYTHON_VERSIONS=(3.12 3.11) + +mkdir -p "${WHEEL_DIR}" + +if [[ -z "${HOST_UV}" ]]; then + if [[ -x /data1/env/uv/bin/uv ]]; then + HOST_UV=/data1/env/uv/bin/uv + elif command -v uv >/dev/null 2>&1; then + HOST_UV="$(command -v uv)" + fi +fi + +echo "============================================================" +echo "1. 准备 uv 二进制 -> ${UV_BIN}" +echo "============================================================" +if [[ -x "${UV_BIN}" ]]; then + echo "已有 ${UV_BIN} ($("${UV_BIN}" --version 2>/dev/null || true))" +else + extracted=0 + # 优先从已经 pull 下来的 DeepSWE 镜像里拷，避免 GitHub + while IFS= read -r image; do + [[ -z "${image}" ]] && continue + if docker image inspect "${image}" >/dev/null 2>&1; then + echo "尝试从 ${image} 提取 uv" + cid="$(docker create --entrypoint /bin/true "${image}" 2>/dev/null || true)" + if [[ -n "${cid}" ]]; then + for src in /root/.local/bin/uv /usr/local/bin/uv /usr/bin/uv; do + rm -f "${UV_BIN}" + if docker cp "${cid}:${src}" "${UV_BIN}" 2>/dev/null && [[ -f "${UV_BIN}" && ! -L "${UV_BIN}" ]]; then + chmod +x "${UV_BIN}" + docker rm "${cid}" >/dev/null + echo "已从镜像 ${src} 提取 uv: $("${UV_BIN}" --version)" + extracted=1 + break 2 + fi + done + docker rm "${cid}" >/dev/null 2>/dev/null || true + fi + fi + done < <(docker images --format '{{.Repository}}:{{.Tag}}' | grep -E 'swe-bench|mars-base' || true) + + if [[ "${extracted}" -eq 0 && -n "${HOST_UV}" ]]; then + echo "复制宿主机 uv: ${HOST_UV}" + cp "${HOST_UV}" "${UV_BIN}" + chmod +x "${UV_BIN}" + extracted=1 + fi + + if [[ "${extracted}" -eq 0 ]]; then + tmp_tar="$(mktemp --suffix=.tar.gz)" + for url in "${UV_MIRRORS[@]}"; do + echo "下载 ${url}" + if curl -L --fail --retry 3 --retry-delay 2 -o "${tmp_tar}" "${url}"; then + tmp_dir="$(mktemp -d)" + tar -C "${tmp_dir}" -xzf "${tmp_tar}" + found="$(find "${tmp_dir}" -type f -name uv -print -quit)" + if [[ -n "${found}" ]]; then + cp "${found}" "${UV_BIN}" + chmod +x "${UV_BIN}" + extracted=1 + echo "已下载 uv: $("${UV_BIN}" --version)" + rm -rf "${tmp_dir}" "${tmp_tar}" + break + fi + rm -rf "${tmp_dir}" + fi + done + rm -f "${tmp_tar}" + fi + + if [[ "${extracted}" -eq 0 ]]; then + echo "ERROR: 无法获得 uv 二进制" + exit 1 + fi +fi + +echo "" +echo "============================================================" +echo "2. 下载 mini-swe-agent wheel（清华/阿里云）" +echo "============================================================" +HOST_PYTHON="${HOST_PYTHON:-}" +if [[ -z "${HOST_PYTHON}" ]]; then + if [[ -x /data1/env/conda/envs/syy/bin/python ]]; then + HOST_PYTHON=/data1/env/conda/envs/syy/bin/python + else + HOST_PYTHON="$(command -v python3)" + fi +fi + +download_ok=0 +for pyver in "${PYTHON_VERSIONS[@]}"; do + abi="cp${pyver//./}" + for index in "${PIP_INDEXES[@]}"; do + echo "pip download mini-swe-agent py${pyver} ${index}" + if "${HOST_PYTHON}" -m pip download mini-swe-agent \ + -d "${WHEEL_DIR}" \ + --python-version "${pyver}" \ + --implementation cp \ + --abi "${abi}" \ + --platform manylinux2014_x86_64 \ + --only-binary=:all: \ + -i "${index}" \ + --default-timeout=120; then + download_ok=1 + break + fi + echo "该源失败，尝试下一个..." + done +done + +if [[ "${download_ok}" -eq 0 ]]; then + echo "ERROR: mini-swe-agent wheel 下载失败" + exit 1 +fi + +wheel_count="$(find "${WHEEL_DIR}" -name '*.whl' | wc -l | tr -d ' ')" +echo "wheel 数量: ${wheel_count}" +if [[ "${wheel_count}" -lt 5 ]]; then + echo "ERROR: wheel 过少，拒绝写 .offline_strict" + exit 1 +fi +touch "${VENDOR_DIR}/.offline_strict" + +echo "" +echo "============================================================" +echo "3. 可选：LiteLLM 价格表（失败可忽略，构建时只是 warning）" +echo "============================================================" +COST_JSON="${VENDOR_DIR}/model_prices_and_context_window.json" +if [[ ! -s "${COST_JSON}" ]]; then + curl -L --fail --retry 2 -o "${COST_JSON}" \ + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" \ + || rm -f "${COST_JSON}" +fi + +echo "" +echo "完成。离线目录: ${VENDOR_DIR}" +ls -lh "${UV_BIN}" +ls "${WHEEL_DIR}" | head +echo "" +echo "评测容器会挂载 evalstone；DeepSWEAdapter 会自动 COPY 该目录进 Pier 构建上下文。" +echo "下一步仍建议先: bash bash/images_load/preload_deep_swe_images.sh" diff --git a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py index d042556..36f9f91 100644 --- a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py +++ b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py @@ -134,6 +134,9 @@ class DeepSWEAdapter(AgentAdapter): def _run_pier_job(self, model: Model, sample: Sample) -> Dict[str, Any]: check_import('pier', extra='deep_swe', raise_error=True, feature_name=self.pretty_name) + from .pier_offline_install import apply_pier_offline_install_patch + + apply_pier_offline_install_patch() from pier.job import Job from pier.models.job.config import JobConfig @@ -265,6 +268,7 @@ integrates it through Pier and runs each benchmark sample as one Pier Python API - Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API - If `TaskConfig.model` has no slash, EvalScope prefixes `pier_model_prefix` (default `openai/`) for Pier/LiteLLM - API base/key are forwarded as `OPENAI_API_BASE` / `OPENAI_API_KEY` from TaskConfig +- Optional offline Pier agent install: run `bash/images_load/preload_deep_swe_agent_offline.sh` so sandbox builds COPY local uv/wheels instead of curling GitHub """, dataset_id=DEFAULT_MODELSCOPE_DATASET_ID, eval_split='test', diff --git a/evalscope/evalscope/benchmarks/deep_swe/pier_offline_install.py b/evalscope/evalscope/benchmarks/deep_swe/pier_offline_install.py new file mode 100644 index 0000000..5c2c72f --- /dev/null +++ b/evalscope/evalscope/benchmarks/deep_swe/pier_offline_install.py @@ -0,0 +1,214 @@ +"""Patch Pier mini-swe-agent Docker builds to use a host-side uv/wheel cache. + +Pier's generated Dockerfile always runs ``curl https://astral.sh/uv/...`` and +``uv tool install mini-swe-agent`` inside every task image. Those steps hit +GitHub/PyPI on each of the ~113 DeepSWE bases (layer cache does not share +across different ``FROM`` images). + +This module: + +1. Copies ``bash/images_load/offline_pier_agent`` into the compose build context +2. Rewrites the install RUN to use the local ``uv`` binary and wheelhouse +""" + +from __future__ import annotations + +import os +import re +import shutil +from pathlib import Path + +from evalscope.utils.logger import get_logger + +logger = get_logger() + +_PATCHED = False +_OFFLINE_MARKER = 'pier-offline' + + +def default_offline_dir() -> Path: + env = os.environ.get('PIER_OFFLINE_AGENT_DIR', '').strip() + if env: + return Path(env) + # .../evalstone/evalscope/evalscope/benchmarks/deep_swe/this.py + return Path(__file__).resolve().parents[4] / 'bash' / 'images_load' / 'offline_pier_agent' + + +def vendor_ready(vendor: Path) -> bool: + if not vendor.is_dir(): + return False + has_uv = (vendor / 'uv').is_file() + has_wheels = any(vendor.joinpath('wheels').glob('*.whl')) + return has_uv or has_wheels + + +def _offline_agent_run(original_run: str) -> str: + version_spec = '' + match = re.search(r'uv tool install mini-swe-agent(==\S+)?', original_run) + if match and match.group(1): + version_spec = match.group(1) + + extra_install = '' + extra_match = re.search( + r'uv pip install --python "\$python_bin" [^\n]+', + original_run, + ) + if extra_match: + extra_install = ( + extra_match.group(0).replace( + 'uv pip install', + 'uv pip install --find-links /tmp/pier-offline/wheels ' + '--index-url https://pypi.tuna.tsinghua.edu.cn/simple', + ) + + '\n' + ) + + return f""" +set -euo pipefail +OFFLINE=/tmp/pier-offline +export PATH="$HOME/.local/bin:$PATH" +mkdir -p "$HOME/.local/bin" +if ! command -v uv >/dev/null 2>&1; then + if [ -x "$OFFLINE/uv" ]; then + cp "$OFFLINE/uv" "$HOME/.local/bin/uv" + chmod +x "$HOME/.local/bin/uv" + else + echo "ERROR: uv missing in image and $OFFLINE/uv not copied into build context" >&2 + exit 1 + fi +fi +if ! grep -q 'export PATH="$HOME/.local/bin:$PATH"' "$HOME/.bashrc" 2>/dev/null; then + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$HOME/.bashrc" +fi +if [ -f "$HOME/.local/bin/env" ]; then + source "$HOME/.local/bin/env" +fi + +UV_PKG_ARGS=() +if ls "$OFFLINE/wheels"/*.whl >/dev/null 2>&1; then + UV_PKG_ARGS+=(--find-links "$OFFLINE/wheels") + if [ -f "$OFFLINE/.offline_strict" ]; then + UV_PKG_ARGS+=(--offline --no-index) + else + UV_PKG_ARGS+=(--index-url https://pypi.tuna.tsinghua.edu.cn/simple) + fi +else + UV_PKG_ARGS+=(--index-url https://pypi.tuna.tsinghua.edu.cn/simple) +fi +uv --python-preference only-system tool install mini-swe-agent{version_spec} "${{UV_PKG_ARGS[@]}}" + +python_bin="$(head -n 1 "$(command -v mini-swe-agent)" | sed 's/^#!//')" +{extra_install} +"$python_bin" <<'PY' +import json +import sys +from importlib.resources import files +from pathlib import Path +from urllib.request import urlopen + +candidates = [ + Path("/tmp/pier-offline/model_prices_and_context_window.json"), +] +url = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +path = files("litellm").joinpath("model_prices_and_context_window_backup.json") + +try: + data = None + for local in candidates: + if local.is_file(): + data = json.loads(local.read_text(encoding="utf-8")) + break + if data is None: + with urlopen(url, timeout=20) as response: + data = json.loads(response.read().decode("utf-8")) + if not isinstance(data, dict) or len(data) < 1000: + raise ValueError( + "unexpected LiteLLM model cost map shape: " + f"{{type(data).__name__}}, " + f"{{len(data) if isinstance(data, dict) else 'n/a'}} entries" + ) + path.write_text(json.dumps(data, separators=(",", ":")), encoding="utf-8") +except Exception as exc: + print( + f"Warning: failed to refresh LiteLLM model cost map backup: {{exc}}", + file=sys.stderr, + ) +PY + +mini-swe-agent --help +""" + + +def apply_pier_offline_install_patch() -> bool: + """Monkeypatch Pier so DeepSWE sandbox builds do not curl GitHub for uv.""" + global _PATCHED + if _PATCHED: + return vendor_ready(default_offline_dir()) + + vendor = default_offline_dir() + if not vendor_ready(vendor): + logger.warning( + 'DeepSWE Pier offline agent cache not found at %s; ' + 'sandbox builds will still download uv from GitHub. ' + 'Run bash/images_load/preload_deep_swe_agent_offline.sh first.', + vendor, + ) + _PATCHED = True + return False + + from pier.agents.installed.mini_swe_agent import MiniSweAgent + from pier.environments import agent_setup + from pier.environments.docker import docker as docker_mod + + orig_install_spec = MiniSweAgent.install_spec + orig_write = agent_setup.write_agent_dockerfile + + def patched_install_spec(self): + spec = orig_install_spec(self) + for step in spec.steps: + if 'astral.sh/uv' in step.run or 'uv tool install mini-swe-agent' in step.run: + step.run = _offline_agent_run(step.run) + return spec + + def patched_write_agent_dockerfile( + *, + build_dir: Path, + source_environment_dir: Path, + prebuilt_image_name: str | None, + install, + user, + ): + dest = Path(build_dir) / _OFFLINE_MARKER + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree( + vendor, + dest, + ignore=shutil.ignore_patterns('__pycache__', '*.tmp', '.git'), + ) + path = orig_write( + build_dir=build_dir, + source_environment_dir=source_environment_dir, + prebuilt_image_name=prebuilt_image_name, + install=install, + user=user, + ) + text = path.read_text(encoding='utf-8') + if f'COPY {_OFFLINE_MARKER} ' not in text: + lines = text.splitlines(True) + out = [] + inserted = False + for line in lines: + out.append(line) + if not inserted and line.startswith('FROM '): + out.append(f'COPY {_OFFLINE_MARKER} /tmp/pier-offline\n') + inserted = True + path.write_text(''.join(out), encoding='utf-8') + return path + + MiniSweAgent.install_spec = patched_install_spec + agent_setup.write_agent_dockerfile = patched_write_agent_dockerfile + docker_mod.write_agent_dockerfile = patched_write_agent_dockerfile + _PATCHED = True + logger.info('Patched Pier mini-swe-agent install to use offline cache %s', vendor) + return True