diff --git a/evalharness/cli.py b/evalharness/cli.py index d1f9f61..000ede1 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -546,11 +546,17 @@ def _compose_judge_spec(args): url = getattr(args, 'judge_api_url', '') or '' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) - if judge and not url and '/' not in judge: - raise SystemExit( - f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' - f'spec like openai/http://host:8000/v1?{judge})') - if url and judge and '/' not in judge: + if judge and not url: + if '/' not in judge and '?' not in judge: + raise SystemExit( + f'--judge-model {judge!r} needs --judge-api-url (or pass a ' + f'full spec like openai/http://host:8000/v1?model)') + return judge # full legacy spec, used as-is + if url and judge: + # url+model ALWAYS combines (same as the main model flags): a bare + # model name may itself contain slashes (/data/hf_models/...), which + # the old 'no slash' heuristic misread as a full spec and passed + # through un-prefixed judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None @@ -644,11 +650,17 @@ def _compose_judge_spec(args): url = getattr(args, 'judge_api_url', '') or '' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) - if judge and not url and '/' not in judge: - raise SystemExit( - f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' - f'spec like openai/http://host:8000/v1?{judge})') - if url and judge and '/' not in judge: + if judge and not url: + if '/' not in judge and '?' not in judge: + raise SystemExit( + f'--judge-model {judge!r} needs --judge-api-url (or pass a ' + f'full spec like openai/http://host:8000/v1?model)') + return judge # full legacy spec, used as-is + if url and judge: + # url+model ALWAYS combines (same as the main model flags): a bare + # model name may itself contain slashes (/data/hf_models/...), which + # the old 'no slash' heuristic misread as a full spec and passed + # through un-prefixed judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None