From dc8d4605ac68666546d8cc64b7b9f16ab7ce8ccd Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 08:07:27 +0000 Subject: [PATCH] Judge: fail fast on missing --judge-api-url; build only when needed A bare --judge-model without --judge-api-url produced a malformed spec that exploded deep inside run_eval -- and it did so for longbench_v2, which does not even use a judge. Now the CLI rejects the combination up front, and run_eval constructs the judge adapter only when the recipe's scorers actually include llm_judge. Co-Authored-By: Claude --- evalharness/cli.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/evalharness/cli.py b/evalharness/cli.py index 0bc253a..d1f9f61 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -546,6 +546,10 @@ def _compose_judge_spec(args): url = getattr(args, 'judge_api_url', '') or '' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) + if judge and not url and '/' not in judge: + raise SystemExit( + f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' + f'spec like openai/http://host:8000/v1?{judge})') if url and judge and '/' not in judge: judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None @@ -640,6 +644,10 @@ def _compose_judge_spec(args): url = getattr(args, 'judge_api_url', '') or '' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) + if judge and not url and '/' not in judge: + raise SystemExit( + f'--judge-model {judge!r} needs --judge-api-url (or pass a full ' + f'spec like openai/http://host:8000/v1?{judge})') if url and judge and '/' not in judge: judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None