Compare commits

..

No commits in common. "97a458d9b80745bf329921890f0e785cf283e76e" and "e9b79a2a412a2d8ccc40cad24d0653bb848509e1" have entirely different histories.

14 changed files with 28 additions and 338 deletions

View File

@ -148,18 +148,6 @@ rep.save('gsm8k.report.json')
# notebook / async 环境用 await evalharness.arun(...)
```
### Agent 环境的额外依赖
`tau2_bench` 需要官方引擎(**PyPI 上的 `tau2` 是同名无关项目,别装错**
```bash
pip install -e /path/to/tau2-bench # 本机源码es 仓库 tools/ 下有)
# 或: pip install git+https://github.com/sierra-research/tau2-bench
```
`swe_bench_verified` 需要 `pip install swebench` + 逐题 Docker 镜像(见 §8
`bfcl_v3` / `general_fc` 无额外依赖。
## 5. 参数速查
| 参数 | 作用 |

View File

@ -546,17 +546,7 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url:
if '/' not in judge and '?' not in judge:
raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a '
f'full spec like openai/http://host:8000/v1?model)')
return judge # full legacy spec, used as-is
if url and judge:
# url+model ALWAYS combines (same as the main model flags): a bare
# model name may itself contain slashes (/data/hf_models/...), which
# the old 'no slash' heuristic misread as a full spec and passed
# through un-prefixed
if url and judge and '/' not in judge:
judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None
@ -650,17 +640,7 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url:
if '/' not in judge and '?' not in judge:
raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a '
f'full spec like openai/http://host:8000/v1?model)')
return judge # full legacy spec, used as-is
if url and judge:
# url+model ALWAYS combines (same as the main model flags): a bare
# model name may itself contain slashes (/data/hf_models/...), which
# the old 'no slash' heuristic misread as a full spec and passed
# through un-prefixed
if url and judge and '/' not in judge:
judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None
@ -765,9 +745,6 @@ def _cmd_eval_run(args) -> int:
try:
if _shared_reporter is not None and total_runs > 1:
_shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]')
if _shared_reporter is not None:
_shared_reporter.begin_bench(name) # clear the previous
# bench's stale scoring bar before this one's dataset load
def _emit(msg, _i=i, _n=name):
if _shared_reporter is not None:
@ -788,12 +765,8 @@ def _cmd_eval_run(args) -> int:
# YAML config: per-bench generation params, AUTO-LOADED
# (single .yaml in config/ = the default; --config overrides)
bench_cfg = _load_bench_cfg(args, name)
# env: per-bench from YAML, CLI --env as fallback/default
_env_cfg = bench_cfg.pop('env', '') or ''
if _env_cfg:
args.env = _env_cfg
# strip non-generation keys (they go to run_eval kwargs)
for k in ('judge', 'judge_url', 'max_turns',
for k in ('judge', 'judge_url', 'env', 'max_turns',
'limit', 'limit_per_task', 'concurrency'):
bench_cfg.pop(k, None)
@ -939,8 +912,7 @@ def _cmd_eval_run(args) -> int:
'max': round(max(_scores), 4),
'std': round(_var ** 0.5, 4),
}
_primary = next((k for k in report.metrics
if k != 'extraction_failure_rate'), '')
_primary = next(iter(report.metrics), '')
if _primary:
report.metrics[f'{_primary}_last_run'] = report.metrics[_primary]
report.metrics[_primary] = _mean
@ -981,11 +953,7 @@ def _cmd_eval_run(args) -> int:
if console is None:
print(render(report, style=args.style))
all_reports.append(report)
# primary metric = first REAL metric; extraction_failure_rate
# is diagnostics (mrcr showed '0.0% extraction_failure_rate' as
# its score in the summary because dict order put it first)
primary = next((k for k in report.metrics
if k != 'extraction_failure_rate'), '')
primary = next(iter(report.metrics), '')
groups = {k: v for k, v in report.metric_groups.items()
if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')}

View File

@ -54,9 +54,7 @@ openai_mrcr:
max_input_tokens: 128000
bfcl_v3:
max_tokens: 4096
env: bfcl_mock # agent 模式:工具调用轨迹 + 官方 AST 判分
general_fc:
max_tokens: 4096
tau2_bench:
max_tokens: 16384
env: tau2_official # agent 模式:官方引擎

View File

@ -22,11 +22,7 @@ def swe_bench_verified():
instance_id = record['instance_id']
# image naming: local swebench/ set uses {repo}_1776_{repo}-{num}; the
# newer official layout is sweb.eval.x86_64.{repo}__{repo}-{num}
# OFFICIAL docker layout: repo-level BASE images
# (swebench/sweb.eval.x86_64.{repo}) -- per-instance images are
# NEVER published; the harness builds them on top of the base
_repo = instance_id.rsplit('-', 1)[0]
img = f'swebench/sweb.eval.x86_64.{_repo}:latest'
img = f'sweb.eval.x86_64.{instance_id}'
if '__' in instance_id:
repo, num = instance_id.rsplit('-', 1)
r = repo.split('__')[0]

View File

@ -13,7 +13,6 @@ Supported sources:
import csv
import json
import os
import subprocess
import sys
import time
import re
@ -262,36 +261,6 @@ def _download_with_progress(resp, out, filename: str) -> None:
out.write(chunk)
def _aria2_fetch(url: str, dest_dir: Path, name: str) -> bool:
"""Multi-connection fetch via aria2c when available.
The mirror's CDN assigns wildly different edges per connection (a
190MB file ran at 6KB/s on one connection and 4.8MB/s on a fresh
one). aria2c splits the file into segments -- each segment rolls its
own edge dice -- and --lowest-speed-limit self-heals a stalled piece
by re-opening it. Returns False (caller falls back to urllib) when
aria2c is missing or fails.
"""
import shutil as _sh
if not _sh.which('aria2c'):
return False
tmp = dest_dir / (name + '.aria2.part')
cmd = ['aria2c', '-x', '8', '-s', '8', '-k', '4M', '--continue=true',
'--file-allocation=none', '--console-log-level=warn',
'--summary-interval=0', '--retry-wait=3', '--max-tries=5',
'--lowest-speed-limit=50K', '--timeout=20',
'-d', str(dest_dir), '-o', tmp.name, url]
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode == 0 and tmp.exists():
os.replace(tmp, dest_dir / name)
return True
tmp.unlink(missing_ok=True)
tmp2 = dest_dir / (tmp.name + '.aria2')
tmp2.unlink(missing_ok=True)
return False
def _ms_download(repo: str, path: str, dest_dir: Path) -> Path:
"""Download one repo file into the raw cache (content-addressed, reused)."""
dest = dest_dir / os.path.basename(path)
@ -302,8 +271,6 @@ def _ms_download(repo: str, path: str, dest_dir: Path) -> Path:
dest.unlink()
dest_dir.mkdir(parents=True, exist_ok=True)
url = f'{_MS_API}/{repo}/repo?Revision=master&FilePath={path}'
if _aria2_fetch(url, dest_dir, dest.name):
return dest_dir / dest.name
tmp = dest.with_name(dest.name + f'.part-{os.getpid()}')
req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'})
with urllib.request.urlopen(req, timeout=600) as resp, open(tmp, 'wb') as out:
@ -444,8 +411,6 @@ def _hf_download(repo: str, path: str, dest_dir: Path) -> Path:
dest.unlink()
dest_dir.mkdir(parents=True, exist_ok=True)
url = f'{_hf_base()}/datasets/{repo}/resolve/main/{path}'
if _aria2_fetch(url, dest_dir, dest.name):
return dest_dir / dest.name
tmp = dest.with_name(dest.name + f'.part-{os.getpid()}')
req = urllib.request.Request(url, headers={'User-Agent': 'evalharness/0.1'})
with urllib.request.urlopen(req, timeout=1800) as resp, open(tmp, 'wb') as out:
@ -457,32 +422,11 @@ def _hf_download(repo: str, path: str, dest_dir: Path) -> Path:
def _load_from_hf_raw(spec: DatasetSpec, raw_dir: Optional[Path] = None) -> List[Dict[str, Any]]:
import hashlib
# the mirror's tree API degrades at night and returns TRUNCATED listings
# (a repo with 9 parquets listed as [.gitattributes, README.md]); the
# blobs from a previous run are already in the shared store -- use them
blob_dir = get_cache_root() / '.raw' / hashlib.md5(spec.source.encode()).hexdigest()[:10]
def _cached_blobs() -> List[str]:
if not blob_dir.exists():
return []
have = [f.name for f in blob_dir.iterdir()
if os.path.splitext(f)[1] in _SUPPORTED_EXTS]
pref = [f for f in have if f.startswith(spec.subset)] or have
return pref
files = _hf_list_files(spec.source)
selected = _hf_match_files(spec, files) if files else []
if not files:
raise FileNotFoundError(f'no files found on HF dataset {spec.source!r}')
selected = _hf_match_files(spec, files)
if not selected:
cached = _cached_blobs()
if cached:
print(f'· listing unavailable/degraded -- using {len(cached)} '
f'cached file(s) from {blob_dir}', flush=True)
records: List[Dict[str, Any]] = []
for f in sorted(cached):
records.extend(_read_file(str(blob_dir / f)))
if raw_dir is not None:
_link_or_copy_all([blob_dir / f for f in sorted(cached)], raw_dir)
return records
raise FileNotFoundError(
f'no data file for subset={spec.subset!r} split={spec.split!r} in '
f'HF {spec.source!r}. Available (first 10): {files[:10]}'

View File

@ -55,11 +55,8 @@ def bigcodebench():
name='bigcodebench',
extract='code_any',
scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
# official sandbox image (bundles every task's deps);
# its ENTRYPOINT is the official evaluate CLI which
# swallows our runner -> override with plain python3
'image': 'bigcodebench/bigcodebench-evaluate:latest',
'entrypoint': 'python3',
# official sandbox image (bundles every task's deps)
'image': 'bigcodebench/bigcodebench-evaluate:latest', # official hub image, same as evalscope
'sandbox': 'docker', 'timeout_s': 120}},
aggregators={'pass': 'pass_at_k'},
exec_workers=12,

View File

@ -120,15 +120,6 @@ def evaluate(
each sample gets its own container/workdir)."""
result = _shell(sample, pred)
raw = result.raw_prediction
# hybrid-thinking backends sometimes inline the reasoning channel
# into content wrapped in <think>...</think> (or leave a stray
# closer): extractors then fish answers out of reasoning text
# ('3</think>Let me analyze...'). Strip the blocks before extract.
if '<think>' in raw or '</think>' in raw:
import re as _re0
raw = _re0.sub(r'<think>.*?</think>', '', raw, flags=_re0.S)
raw = raw.replace('</think>', '')
try:
if bp is not None and sample.sandbox and sample.sandbox.image:
bp.ensure(sample.sandbox.image) # wait only if this one still pulling

View File

@ -342,8 +342,7 @@ def execution(pred: str, target, sample: Sample, ctx: ScoreContext):
files = harness(sample, pred or '')
result = sbx.exec(files, entry=ctx.params.get('entry', 'main.py'),
timeout_s=ctx.params.get('timeout_s', 30),
image=ctx.params.get('image', ''),
entrypoint=ctx.params.get('entrypoint', ''))
image=ctx.params.get('image', ''))
ok = result.ok
return ({'pass': 1.0} if ok else {'pass': 0.0}), {'pass': {
'exit_code': result.exit_code,

View File

@ -821,8 +821,7 @@ async def run_eval(
status_callback('Scores cached in checkpoint -- replaying '
'(no scorers run; --rescore re-evaluates)')
if judge is None and judge_spec and _records is None \
and _recipe_needs_judge(recipe):
if judge is None and judge_spec and _records is None:
if status_callback:
status_callback('loading judge model')
judge_adapter = _make_adapter(judge_spec, api_key=judge_api_key or api_key)
@ -830,13 +829,6 @@ async def run_eval(
if status_callback:
status_callback('Scoring predictions against the benchmark recipe')
# retarget the bar to scoring IMMEDIATELY (0/N): waiting for the first
# on_scored left the stale GENERATION counters (100%, +N new) on screen
# through minute-long preflights (image pulls) -- reading as 'done'
if progress_reporter is not None and _records is None:
_ss0 = getattr(progress_reporter, 'set_scoring', None)
if _ss0 is not None:
_ss0(0, len(samples))
_meta = {'gen_input_tokens': usage.input_tokens,
'gen_output_tokens': usage.output_tokens,
'gen_total_tokens': usage.total_tokens,
@ -894,20 +886,6 @@ async def run_eval(
return report
def _recipe_needs_judge(recipe) -> bool:
"""Only construct the judge when a scorer actually consumes it --
rule-based benches (longbench_v2 etc.) never touch a judge, and
building one there crashed on malformed specs for no benefit."""
try:
for spec in (recipe.scorers or {}).values():
p = spec if isinstance(spec, dict) else {}
if p.get('name') in ('llm_judge', 'judge'):
return True
except Exception:
pass
return False
def _env_user_adapter(spec: str):
"""Build (once per spec) the separate USER-simulator adapter for env
benches (tau2 strong-user parity mode)."""

View File

@ -162,26 +162,6 @@ class RichTerminalProgress:
self.heartbeat_task.cancel()
self.heartbeat_task = asyncio.create_task(self._heartbeat())
def begin_bench(self, name: str):
"""New benchmark starting: immediately re-label the bar to its
loading phase. Without this the PREVIOUS bench's stale scoring
state (e.g. a failed '0/2') stayed on screen through the whole
dataset download of the next one."""
if self.disabled or self.task_id is None:
return
self.bench_name = name
self._last_phase = 'loading'
self._scoring_for = None
self.started = time.monotonic()
self.inflight = 0
self.restored = 0
self.admitted = None
self.progress.update(
self.task_id,
description=f'[green]{self.bench_tag}{name} · loading[/green]',
total=1, completed=0, new='', rate='0.00', inflight=0,
cur='0s', elapsed='0s', eta='-', retries=0)
def set_scoring(self, done: int, total: int):
"""Retarget the SAME bar to the scoring phase: generation is finished
and its filled 100% state is stale -- now the bar refills with judged

View File

@ -65,8 +65,6 @@ class Sandbox:
mounts: Optional[Dict[str, str]] = None,
timeout_s: int = 60,
image: str = '',
entrypoint: str = '', # docker-only override; accepted for
# signature parity (local runs a plain python)
) -> ExecResult:
"""Run ``python <entry>`` with ``files`` (name->content) in isolation.

View File

@ -45,13 +45,11 @@ def ensure_image(img: str) -> None:
return
except RuntimeError as e:
raise RuntimeError(
f'沙箱镜像 {img!r} 不可用:本地没有,全部镜像源尝试失败。'
f'{str(e)[:150]}\n'
'需要自己加载镜像(任选其一),然后重跑加 --rescore\n'
f' A. 有外网的机器: docker pull {img} && docker save {img} | gzip > img.tgz\n'
f' 拷回本机后: docker load < img.tgz\n'
f' B. 本机网络恢复后: docker pull {img}\n'
'预测都在 checkpoint 里,镜像就位后纯判分即可。') from e
f'sandbox image {img!r} is not available: not local, and every '
f'mirror failed. {str(e)[:200]}. '
'Fix: pull/build it manually (for bigcodebench the official image '
'is bigcodebench/bigcodebench-evaluate:latest), then re-run with '
'--rescore to score the cached predictions.') from e
@register_sandbox('docker')
@ -67,7 +65,6 @@ class DockerSandbox(Sandbox):
mounts: Optional[Dict[str, str]] = None,
timeout_s: int = 60,
image: str = '',
entrypoint: str = '',
) -> ExecResult:
img = image or self.DEFAULT_EXEC_IMAGE
with tempfile.TemporaryDirectory(prefix='eh-sbx-') as host_dir:
@ -93,15 +90,6 @@ class DockerSandbox(Sandbox):
out_host = Path(hpath).expanduser()
out_host.mkdir(parents=True, exist_ok=True)
cmd += ['-v', f'{out_host}:{cpath}:rw']
if entrypoint:
# images with an official-evaluator ENTRYPOINT (bigcodebench:
# python3 -m bigcodebench.evaluate) swallow our runner as
# CLI args -> override it. The entrypoint IS the interpreter
# now: pass only the script path, else 'python3 python
# /work/main.py' tries to open a file named 'python'
cmd += ['--entrypoint', entrypoint]
runner = [f'/work/{entry}']
else:
runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \
else ['sh', f'/work/{entry}']
# Named + retried runs. A timed-out/killed `docker run` only kills

View File

@ -25,8 +25,6 @@ class LocalSandbox(Sandbox):
mounts: Optional[Dict[str, str]] = None,
timeout_s: int = 60,
image: str = '',
entrypoint: str = '', # docker-only override; accepted for
# signature parity (local runs a plain python)
) -> ExecResult:
with tempfile.TemporaryDirectory(prefix='eh-local-') as td:
work = Path(td)

View File

@ -19,25 +19,15 @@ from ..data.dataset import Dataset
# CN mirrors tried in order before/alongside the daemon's configured mirrors.
# Some namespaces (e.g. swebench/*) are blocked by individual CN mirrors, so we
# fall through: daemon default -> 1ms.run -> baidubce -> sjtug.
# Six mainstream fallbacks (trimmed from 16 per user call: the long tail
# died anyway). History on this host: daocloud and 1ms.run have each
# delivered the 9GB bigcodebench image; the rest are the usual suspects.
_CN_MIRROR_FALLBACKS = [
'{img}', # daemon default (uses its own registry-mirrors config)
'docker.m.daocloud.io/{img}',
'docker.1ms.run/{img}',
'docker.1panel.live/{img}',
'mirror.baidubce.com/{img}',
'docker.mirrors.sjtug.sjtu.edu.cn/{img}',
'hub.rat.dev/{img}',
'docker.mirrors.ustc.edu.cn/{img}',
]
# 用户/环境可整体覆盖(逗号分隔模板,{img} 占位)
import os as _os
_env = _os.environ.get('EVALHARNESS_DOCKER_MIRRORS', '').strip()
if _env:
_CN_MIRROR_FALLBACKS = [t.strip() for t in _env.split(',') if t.strip()]
def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
"""Collect distinct sandbox images declared by a dataset's samples."""
@ -103,139 +93,16 @@ def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]:
return pulled
def _pull_one(image: str, verbose: bool = True) -> None:
"""Pull via CN-mirror fallback chain; retag to the canonical name on hit.
A multi-GB pull runs silently for many minutes with capture_output --
which reads as a hang -- so say WHICH source is being tried and how
long it took."""
import sys as _sys
import time as _time
def _pull_one(image: str) -> None:
"""Pull via CN-mirror fallback chain; retag to the canonical name on hit."""
last_err = None
for template in _CN_MIRROR_FALLBACKS:
ref = template.format(img=image)
if verbose:
# the daemon-default entry IS exactly '{img}'; every real
# mirror template also CONTAINS '{img}' -- the old containment
# check matched both and labeled everything docker.io
src = ('docker.io (daemon mirrors)' if template.strip() == '{img}'
else template.split('/{img}')[0])
print(f'· pulling sandbox image {image} via {src} ...',
file=_sys.stderr, flush=True)
t0 = _time.monotonic()
# stream progress: a multi-GB pull with captured output is minutes
# of silence that reads as a hang. Non-TTY docker pull emits clean
# per-layer lines; forward the informative ones (throttled).
# docker 25+ hides per-layer progress when stdout is not a TTY;
# --progress=plain forces the id: Downloading x/y lines our ⏳
# summary parses (without it even a real 9GB/74s pull is silent)
proc = subprocess.Popen(['docker', 'pull', '--progress=plain', ref],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
bufsize=0)
# idle watchdog: a dead mirror emits NOTHING (0 bytes/s for hours);
# the old fixed 3600s timeout let one dead source stall the whole
# chain. 5 minutes of silence = kill and try the next mirror.
import os as _os
import selectors as _sel
_selr = _sel.DefaultSelector()
_selr.register(proc.stdout, _sel.EVENT_READ)
_fd = proc.stdout.fileno()
last_line = ''
last_show = 0.0
layer_prog = {} # layer id -> (done_bytes, total_bytes): the raw
# per-layer events give no sense of OVERALL progress -- sum them
import os as _os
import re as _re
def _bytes(s):
s = s.strip()
mult = {'kB': 1e3, 'KB': 1e3, 'MB': 1e6, 'GB': 1e9, 'B': 1}
for suf, m in mult.items():
if s.endswith(suf):
return int(float(s[:-len(suf)]) * m)
return int(float(s or 0))
# docker progress lines end with \r (carriage-return refresh), not
# \n -- readline() buffered them until a rare \n arrived, so the
# Downloading snapshots (and the ⏳ summary fed by them) never
# surfaced. Split on BOTH terminators, chunked reads.
buf = ''
_lines_iter = iter(lambda: None, 1) # placeholder, replaced below
# no output at all -> dead mirror, switch source. ANY output resets
# the timer (manifest/layer announcements/DOWNLOADING refreshes), so
# this only fires on sources that print nothing. 10s default;
# override with EVALHARNESS_PULL_IDLE_S for slow-negotiating mirrors.
import os as _os2
_IDLE_S = float(_os2.environ.get('EVALHARNESS_PULL_IDLE_S', '300'))
def _iter_lines():
nonlocal buf
while True:
if not _selr.select(timeout=_IDLE_S):
proc.kill()
if verbose:
print(f'· no data for {_IDLE_S:.0f}s -- dead mirror, '
'trying next source ...',
file=_sys.stderr, flush=True)
return
chunk = _os.read(_fd, 4096)
if not chunk:
if buf:
yield buf
return
buf += chunk.decode('utf-8', 'replace')
parts = buf.split('\r') if '\r' in buf else buf.split('\n')
if len(parts) > 1:
for p in parts[:-1]:
yield p
buf = parts[-1]
for line in _iter_lines():
line = line.strip('\n ')
last_line = line
m = _re.match(r'^([0-9a-f]{12}): Downloading.*?([\d.]+[kKMG]?B)/([\d.]+[kKMG]?B)', line)
if m:
layer_prog[m.group(1)] = (_bytes(m.group(2)), _bytes(m.group(3)))
if 'Pull complete' in line:
lid = line.split(':')[0]
if lid in layer_prog:
layer_prog[lid] = (layer_prog[lid][1], layer_prog[lid][1])
now = _time.monotonic()
key_evt = any(k in line for k in
('Pulling from', 'Status', 'error', 'denied'))
if verbose and key_evt:
print(f' {last_line[:100]}', file=_sys.stderr, flush=True)
last_show = now
elif verbose and now - last_show > 3 and layer_prog:
done = sum(v[0] for v in layer_prog.values())
tot = sum(v[1] for v in layer_prog.values())
if tot:
pct = done / tot * 100
big = max(layer_prog.items(), key=lambda kv: kv[1][1] - kv[1][0])
print(f'{done / 1e9:.2f}/{tot / 1e9:.2f} GB ({pct:.0f}%)'
f' · 最大层 {big[0]}: {big[1][0] / 1e9:.2f}/{big[1][1] / 1e9:.2f} GB',
file=_sys.stderr, flush=True)
last_show = now
proc.wait(timeout=3600)
r = subprocess.CompletedProcess(proc.args, proc.returncode,
stdout='', stderr=last_line)
if r.returncode == 0 and verbose:
print(f'✓ image ready in {_time.monotonic() - t0:.0f}s',
file=_sys.stderr, flush=True)
r = subprocess.run(['docker', 'pull', ref], capture_output=True, text=True,
timeout=3600)
if r.returncode == 0:
if ref != image:
# retag BEFORE rmi and VERIFY: a silent tag failure followed
# by rmi deleted the only tag and lost a 342s multi-GB pull
tag = subprocess.run(['docker', 'tag', ref, image],
capture_output=True, text=True)
if tag.returncode != 0:
print(f'!! docker tag {ref} -> {image} failed: '
f'{(tag.stderr or "")[:120]} (keeping the mirror tag)',
file=_sys.stderr, flush=True)
return # do NOT rmi: the mirror tag is the only handle
if ref != image: # retag the mirrored pull to the canonical name
subprocess.run(['docker', 'tag', ref, image], check=False)
subprocess.run(['docker', 'rmi', ref], check=False)
return
last_err = (ref, (r.stderr or '').strip()[:150])