From b3c10d23f24a9e210f1c28872de38356d14ac01f Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Thu, 17 Sep 2026 08:08:13 +0000
Subject: [PATCH] judge spec: url+model always combines (path-shaped model ids)

'/data/hf_models/GLM-5.3-NVFP4' contains slashes, so the bare-name
heuristic ('/' not in judge) misclassified it as a full spec and passed
it through un-prefixed -- resolve_adapter then blew up. With
--judge-api-url given, combination is now unconditional (mirrors the
main model flags); bare names without a url fail fast; full specs
without a url pass through.

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/cli.py | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/evalharness/cli.py b/evalharness/cli.py
index d1f9f61..000ede1 100644
--- a/evalharness/cli.py
+++ b/evalharness/cli.py
@@ -546,11 +546,17 @@ def _compose_judge_spec(args):
     url = getattr(args, 'judge_api_url', '') or ''
     provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
     internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
-    if judge and not url and '/' not in judge:
-        raise SystemExit(
-            f'--judge-model {judge!r} needs --judge-api-url (or pass a full '
-            f'spec like openai/http://host:8000/v1?{judge})')
-    if url and judge and '/' not in judge:
+    if judge and not url:
+        if '/' not in judge and '?' not in judge:
+            raise SystemExit(
+                f'--judge-model {judge!r} needs --judge-api-url (or pass a '
+                f'full spec like openai/http://host:8000/v1?model)')
+        return judge   # full legacy spec, used as-is
+    if url and judge:
+        # url+model ALWAYS combines (same as the main model flags): a bare
+        # model name may itself contain slashes (/data/hf_models/...), which
+        # the old 'no slash' heuristic misread as a full spec and passed
+        # through un-prefixed
         judge = f'{internal}/{url.rstrip("/")}?{judge}'
     return judge or None
 
@@ -644,11 +650,17 @@ def _compose_judge_spec(args):
     url = getattr(args, 'judge_api_url', '') or ''
     provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
     internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
-    if judge and not url and '/' not in judge:
-        raise SystemExit(
-            f'--judge-model {judge!r} needs --judge-api-url (or pass a full '
-            f'spec like openai/http://host:8000/v1?{judge})')
-    if url and judge and '/' not in judge:
+    if judge and not url:
+        if '/' not in judge and '?' not in judge:
+            raise SystemExit(
+                f'--judge-model {judge!r} needs --judge-api-url (or pass a '
+                f'full spec like openai/http://host:8000/v1?model)')
+        return judge   # full legacy spec, used as-is
+    if url and judge:
+        # url+model ALWAYS combines (same as the main model flags): a bare
+        # model name may itself contain slashes (/data/hf_models/...), which
+        # the old 'no slash' heuristic misread as a full spec and passed
+        # through un-prefixed
         judge = f'{internal}/{url.rstrip("/")}?{judge}'
     return judge or None