From d58fdb41980eba43d3b1ab14e6b1455b938aaf65 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 05:39:58 +0000 Subject: [PATCH] Pull chain: 5-minute idle watchdog per mirror A dead mirror emitted zero bytes and the fixed 3600s attempt timeout let it stall the whole chain for an hour (third occurrence tonight: 15m49s frozen at 0 bytes). select() with a 300s idle timeout now kills the attempt and moves to the next mirror; a trickling source (1 line/s Downloading updates) resets the timer and stays alive. Co-Authored-By: Claude --- evalharness/sandbox/prefetch.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/evalharness/sandbox/prefetch.py b/evalharness/sandbox/prefetch.py index e59d24e..c3aa8e7 100644 --- a/evalharness/sandbox/prefetch.py +++ b/evalharness/sandbox/prefetch.py @@ -115,6 +115,15 @@ def _pull_one(image: str, verbose: bool = True) -> None: proc = subprocess.Popen(['docker', 'pull', ref], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) + # idle watchdog: a dead mirror emits NOTHING (0 bytes/s for hours); + # the old fixed 3600s timeout let one dead source stall the whole + # chain. 5 minutes of silence = kill and try the next mirror. + import os as _os + import selectors as _sel + + _selr = _sel.DefaultSelector() + _selr.register(proc.stdout, _sel.EVENT_READ) + _fd = proc.stdout.fileno() last_line = '' last_show = 0.0 layer_prog = {} # layer id -> (done_bytes, total_bytes): the raw @@ -134,20 +143,27 @@ def _pull_one(image: str, verbose: bool = True) -> None: # \n -- readline() buffered them until a rare \n arrived, so the # Downloading snapshots (and the ⏳ summary fed by them) never # surfaced. Split on BOTH terminators, chunked reads. - import io as _io - stream = _io.TextIOWrapper(proc.stdout, errors='replace') buf = '' _lines_iter = iter(lambda: None, 1) # placeholder, replaced below + _IDLE_S = 300 # no output for 5 minutes -> dead mirror + def _iter_lines(): nonlocal buf while True: - chunk = stream.read(512) + if not _selr.select(timeout=_IDLE_S): + proc.kill() + if verbose: + print(f'· no data for {_IDLE_S // 60}min -- dead ' + 'mirror, trying next source ...', + file=_sys.stderr, flush=True) + return + chunk = _os.read(_fd, 4096) if not chunk: if buf: yield buf return - buf += chunk + buf += chunk.decode('utf-8', 'replace') parts = buf.split('\r') if '\r' in buf else buf.split('\n') if len(parts) > 1: for p in parts[:-1]: