Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

openai-pool: allow plain-URL members (no port range required)

Browse Source 
--auto-concurrency wraps a single endpoint as a 1-member pool; the
spec parser rejected it with 'needs a {start..end} port range' and
killed every benchmark. Range-less comma segments are now plain
members; only a fully empty endpoint list errors.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 04:05:18 +00:00
parent 6ad68bdfc0
commit 9049bd683d
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
evalharness/model/runner.py
View File

@ -852,13 +852,12 @@ def _make_adapter(spec: str, api_key: str = '') -> ModelAdapter:
from .pool import pooled from .pool import pooled
rest = spec[len('openai-pool/'):] rest = spec[len('openai-pool/'):]
ms = __import__('re').findall(r'\{(\d+)\.\.(\d+)\}', rest)
if not ms:
raise ValueError("openai-pool needs a {start..end} port range")
base_url, _, model = rest.partition('?') base_url, _, model = rest.partition('?')
# expand EACH comma-separated segment's OWN range independently -- # expand EACH comma-separated segment's OWN range independently --
# a global sub(count=1) would keep replacing only the FIRST range # a global sub(count=1) would keep replacing only the FIRST range
# and emit URLs with literal '{8200..8203}' in later segments # and emit URLs with literal '{8200..8203}' in later segments.
# Segments WITHOUT a range are plain members: --auto-concurrency
# wraps a single endpoint as a 1-member pool to get the gate.
specs = [] specs = []
for seg in base_url.split(','): for seg in base_url.split(','):
seg = seg.strip() seg = seg.strip()
@ -870,6 +869,9 @@ def _make_adapter(spec: str, api_key: str = '') -> ModelAdapter:
specs.append(f'openai/{u}?{model}') specs.append(f'openai/{u}?{model}')
elif seg: elif seg:
specs.append(f'openai/{seg}?{model}') specs.append(f'openai/{seg}?{model}')
if not specs:
raise ValueError('openai-pool needs at least one endpoint '
'(plain URL or {start..end} port range)')
adapter = pooled(specs, api_key=api_key) if api_key else pooled(specs) adapter = pooled(specs, api_key=api_key) if api_key else pooled(specs)
elif re.fullmatch(r'mock[-:](boxed|oracle|fc|tool|echo|const)?', spec): elif re.fullmatch(r'mock[-:](boxed|oracle|fc|tool|echo|const)?', spec):
# mock-boxed (preferred) == legacy mock:boxed; bare 'mock' == echo. # mock-boxed (preferred) == legacy mock:boxed; bare 'mock' == echo.