Pull chain: 5-minute idle watchdog per mirror

A dead mirror emitted zero bytes and the fixed 3600s attempt timeout
let it stall the whole chain for an hour (third occurrence tonight:
15m49s frozen at 0 bytes). select() with a 300s idle timeout now kills
the attempt and moves to the next mirror; a trickling source (1 line/s
Downloading updates) resets the timer and stays alive.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 05:39:58 +00:00
parent 3ae1578325
commit d58fdb4198

View File

@ -115,6 +115,15 @@ def _pull_one(image: str, verbose: bool = True) -> None:
proc = subprocess.Popen(['docker', 'pull', ref],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
bufsize=0)
# idle watchdog: a dead mirror emits NOTHING (0 bytes/s for hours);
# the old fixed 3600s timeout let one dead source stall the whole
# chain. 5 minutes of silence = kill and try the next mirror.
import os as _os
import selectors as _sel
_selr = _sel.DefaultSelector()
_selr.register(proc.stdout, _sel.EVENT_READ)
_fd = proc.stdout.fileno()
last_line = ''
last_show = 0.0
layer_prog = {} # layer id -> (done_bytes, total_bytes): the raw
@ -134,20 +143,27 @@ def _pull_one(image: str, verbose: bool = True) -> None:
# \n -- readline() buffered them until a rare \n arrived, so the
# Downloading snapshots (and the ⏳ summary fed by them) never
# surfaced. Split on BOTH terminators, chunked reads.
import io as _io
stream = _io.TextIOWrapper(proc.stdout, errors='replace')
buf = ''
_lines_iter = iter(lambda: None, 1) # placeholder, replaced below
_IDLE_S = 300 # no output for 5 minutes -> dead mirror
def _iter_lines():
nonlocal buf
while True:
chunk = stream.read(512)
if not _selr.select(timeout=_IDLE_S):
proc.kill()
if verbose:
print(f'· no data for {_IDLE_S // 60}min -- dead '
'mirror, trying next source ...',
file=_sys.stderr, flush=True)
return
chunk = _os.read(_fd, 4096)
if not chunk:
if buf:
yield buf
return
buf += chunk
buf += chunk.decode('utf-8', 'replace')
parts = buf.split('\r') if '\r' in buf else buf.split('\n')
if len(parts) > 1:
for p in parts[:-1]: