Sandbox: reap timed-out containers, retry daemon-side failures; scoring milestones

- docker exec: named containers; a timed-out/killed 'docker run' only
  kills the CLI client while the container lives on (--rm fires on
  EXIT) -- rm -f the name on timeout/interrupt so runs stop leaking
- exit 125 = daemon-side failure, not model failure: retry up to 2x
  (a bloated daemon was turning healthy samples into pass=0)
- scoring milestones: first completion logs immediately, then every
  ~5% (10% was too sparse when docker is slow: minutes of silence
  right after the 'scoring' phase starts, looks hung)

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -721,9 +721,12 @@ def _cmd_eval_run(args) -> int:
                     'generating 100%' and looks hung)."""
                     if _reporter is not None:
                         _reporter.set_phase(f'scoring {done}/{total_s}')
-                    # milestone lines: pipes/logs without a live bar see movement
-                    if _cb and total_s and (done % max(1, total_s // 10) == 0
-                                            or done == total_s):
+                    # milestone lines: FIRST completion reports immediately
+                    # (docker-slow runs otherwise look frozen for minutes),
+                    # then every ~5%
+                    step = max(1, total_s // 20) if total_s else 1
+                    if _cb and (done == 1 or done % step == 0
+                                or done == total_s):
                         _cb(f'Scoring {done}/{total_s} samples')
                 _gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})}
                 # max_input_tokens must be a SEPARATE run_eval param (it drives

evalharness/sandbox/docker.py

@@ -66,13 +66,32 @@ class DockerSandbox(Sandbox):
                     cmd += ['-v', f'{out_host}:{cpath}:rw']
             runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \
                 else ['sh', f'/work/{entry}']
-            cmd += [img, *runner]
+            # Named + retried runs. A timed-out/killed `docker run` only kills
+            # the CLI client -- the container lives on (--rm fires on EXIT)
+            # and leaked containers slowly drown the daemon; the name gives
+            # us something to rm -f. Exit 125 is a DAEMON-side failure (shim
+            # error etc.), not the model's code failing -- scoring it pass=0
+            # would corrupt results, so retry it.
+            import uuid
+
             t0 = time.time()
-            try:
-                proc = _run(cmd, timeout=timeout_s + 30)
-            except subprocess.TimeoutExpired:
-                return ExecResult(exit_code=-1, timed_out=True, duration_s=timeout_s,
-                                  error=f'sandbox timeout after {timeout_s}s')
+            proc = None
+            for attempt in range(3):
+                cname = f'eh-exec-{uuid.uuid4().hex[:10]}'
+                full = cmd + ['--name', cname, img, *runner]
+                try:
+                    proc = _run(full, timeout=timeout_s + 30)
+                except subprocess.TimeoutExpired:
+                    _run(['docker', 'rm', '-f', cname])  # CLI died, container didn't
+                    return ExecResult(exit_code=-1, timed_out=True, duration_s=timeout_s,
+                                      error=f'sandbox timeout after {timeout_s}s')
+                except BaseException:  # Ctrl+C / kill: reap, then propagate
+                    _run(['docker', 'rm', '-f', cname])
+                    raise
+                if proc.returncode != 125 or attempt == 2:
+                    break
+                _run(['docker', 'rm', '-f', cname])  # clear any husk; fresh name next try
+                time.sleep(2 * (attempt + 1))        # give the daemon a beat
             return ExecResult(
                 exit_code=proc.returncode,
                 stdout=proc.stdout,