Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

run_one: exactly 1 attempt (adapter handles all retries internally)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 07:07:22 +00:00
parent 55b2beab54
commit d64b236540
1 changed files with 9 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
evalharness/model/runner.py
View File

@ -419,18 +419,15 @@ async def generate_predictions(
# backoff so one ConnectError burst cannot kill the whole batch -- # backoff so one ConnectError burst cannot kill the whole batch --
# the adapter already retries 5xx/429 and the pool fails over per # the adapter already retries 5xx/429 and the pool fails over per
# instance; this is the last line of defense around asyncio.gather # instance; this is the last line of defense around asyncio.gather
for attempt in range(3): # NO outer retry: the adapter retries internally; a second loop
try: # here multiplied worst-case time (42+ attempts before this fix).
pred = await one(s) # One pass, one result or one error.
break try:
except Exception: pred = await one(s)
if attempt == 2: except Exception:
if progress_reporter is not None: if progress_reporter is not None:
progress_reporter.advance(success=False) progress_reporter.advance(success=False)
raise raise
# minute-scale backoff: cluster routes flap in multi-minute
# bursts; short retries exhaust inside one dead window
await asyncio.sleep((10, 20, 40)[attempt])
if ckpt_store is not None: if ckpt_store is not None:
ckpt_store.append(keys[i], pred) ckpt_store.append(keys[i], pred)
return i, pred return i, pred