diff --git a/evalharness/agent/envs/swe_agentic.py b/evalharness/agent/envs/swe_agentic.py index cbd78c1..c840d0a 100644 --- a/evalharness/agent/envs/swe_agentic.py +++ b/evalharness/agent/envs/swe_agentic.py @@ -13,12 +13,56 @@ import asyncio import json import re import subprocess +import sys import uuid from typing import Any, Dict from ...data.sample import ChatMessage, Sample from ..loop import Environment, register_env +# ---- image lifecycle tracking ------------------------------------------- +# per-instance swe images are BIG (1-4GB each, 500 total ~2TB): every +# image THIS process pulled is registered here and released at exit +# (normal end, Ctrl+C kill, crash) so a run never strands its footprint. +import atexit as _atexit + +_IMAGES_PULLED: list = [] + + +def _release_pulled_images() -> None: + import os as _os + + if not _IMAGES_PULLED or _os.environ.get('EVALHARNESS_KEEP_SWE_IMAGES'): + return + # in-use containers first (docker refuses rmi otherwise) + try: + _docker(['ps', '-aq', '--filter', 'name=eh-swe-'], timeout=30) + except Exception: + pass + try: + _docker(['ps', '-aq', '--filter', 'ancestor=none'], timeout=30) + except Exception: + pass + import subprocess as _sp + + try: + ids = _sp.run(['docker', 'ps', '-aq', '--filter', 'name=eh-swe'], + capture_output=True, text=True, timeout=30).stdout.split() + if ids: + _docker(['rm', '-f'] + ids, timeout=120) + except Exception: + pass + for img in _IMAGES_PULLED: + try: + _docker(['rmi', '-f', img], timeout=120) + print(f'ยท released swe image {img}', file=sys.stderr, flush=True) + except Exception: + pass + + +_atexit.register(_release_pulled_images) +# ------------------------------------------------------------------------ + SENTINEL = 'COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT' # mini-swe-agent swebench.yaml contract (verbatim sections that matter) @@ -123,10 +167,13 @@ class SWEAgenticEnvironment(Environment): # output and its 120s timeout kills runs on slow mirrors from ...sandbox.docker import ensure_image + had_it = _docker(['image', 'inspect', image]).returncode == 0 try: ensure_image(image) except RuntimeError as e: raise RuntimeError(f'image {image} unavailable: {str(e)[:150]}') + if not had_it and image not in _IMAGES_PULLED: + _IMAGES_PULLED.append(image) # we pulled it -> we release it name = f'eh-swe-{uuid.uuid4().hex[:10]}' r = _docker(['run', '-d', '--name', name, '-w', '/testbed',