Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

judge spec: url+model always combines (path-shaped model ids)

Browse Source 
'/data/hf_models/GLM-5.3-NVFP4' contains slashes, so the bare-name
heuristic ('/' not in judge) misclassified it as a full spec and passed
it through un-prefixed -- resolve_adapter then blew up. With
--judge-api-url given, combination is now unconditional (mirrors the
main model flags); bare names without a url fail fast; full specs
without a url pass through.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 08:08:13 +00:00
parent fe902db330
commit b3c10d23f2
1 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
evalharness/cli.py
View File

@ -546,11 +546,17 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or '' url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url and '/' not in judge: if judge and not url:
if '/' not in judge and '?' not in judge:
raise SystemExit( raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' f'--judge-model {judge!r} needs --judge-api-url (or pass a '
f'spec like openai/http://host:8000/v1?{judge})') f'full spec like openai/http://host:8000/v1?model)')
if url and judge and '/' not in judge: return judge # full legacy spec, used as-is
if url and judge:
# url+model ALWAYS combines (same as the main model flags): a bare
# model name may itself contain slashes (/data/hf_models/...), which
# the old 'no slash' heuristic misread as a full spec and passed
# through un-prefixed
judge = f'{internal}/{url.rstrip("/")}?{judge}' judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None return judge or None
@ -644,11 +650,17 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or '' url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url and '/' not in judge: if judge and not url:
if '/' not in judge and '?' not in judge:
raise SystemExit( raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' f'--judge-model {judge!r} needs --judge-api-url (or pass a '
f'spec like openai/http://host:8000/v1?{judge})') f'full spec like openai/http://host:8000/v1?model)')
if url and judge and '/' not in judge: return judge # full legacy spec, used as-is
if url and judge:
# url+model ALWAYS combines (same as the main model flags): a bare
# model name may itself contain slashes (/data/hf_models/...), which
# the old 'no slash' heuristic misread as a full spec and passed
# through un-prefixed
judge = f'{internal}/{url.rstrip("/")}?{judge}' judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None return judge or None