Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Judge: fail fast on missing --judge-api-url; build only when needed

Browse Source 
A bare --judge-model without --judge-api-url produced a malformed spec
that exploded deep inside run_eval -- and it did so for longbench_v2,
which does not even use a judge. Now the CLI rejects the combination
up front, and run_eval constructs the judge adapter only when the
recipe's scorers actually include llm_judge.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 08:07:27 +00:00
parent 3f2ca8bfd5
commit dc8d4605ac
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
evalharness/cli.py
View File

@ -546,6 +546,10 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or '' url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url and '/' not in judge:
raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a full '
f'spec like openai/http://host:8000/v1?{judge})')
if url and judge and '/' not in judge: if url and judge and '/' not in judge:
judge = f'{internal}/{url.rstrip("/")}?{judge}' judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None return judge or None
@ -640,6 +644,10 @@ def _compose_judge_spec(args):
url = getattr(args, 'judge_api_url', '') or '' url = getattr(args, 'judge_api_url', '') or ''
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
if judge and not url and '/' not in judge:
raise SystemExit(
f'--judge-model {judge!r} needs --judge-api-url (or pass a full '
f'spec like openai/http://host:8000/v1?{judge})')
if url and judge and '/' not in judge: if url and judge and '/' not in judge:
judge = f'{internal}/{url.rstrip("/")}?{judge}' judge = f'{internal}/{url.rstrip("/")}?{judge}'
return judge or None return judge or None