diff --git a/README.md b/README.md index 33aa2c9..ec59054 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ evalharness eval run gsm8k \ | `--model NAME` | 服务端模型名(配合 `--api-url`);或直接给完整 spec:`openai/http://h:8000/v1?qwen3-8b` | | `--provider {openai-chat,openai-pool}` | 协议/提供方,默认 `openai-chat`;端点池用 `openai-pool` | | `--judge NAME` + `--judge-api-url URL` | LLM-judge 模型(hle / simple_qa / imo 需要);也可单给完整 spec `--judge openai/http://...?m` | +| `--judge-provider {openai-chat,openai-pool}` | judge 协议;多 judge 端点负载均衡用 `openai-pool` | | `--profile NAME` | 命名生成参数集(内置 `dp4-nothink`、`qwen3-es-parity`、`t1-short`,或任意 `@register_gen_profile` 名);优先级:插件默认 < profile 默认 < profile 单 bench 覆盖 < 显式参数 | | `--disable-thinking` | 发送 `enable_thinking=false`(Qwen3 类思考模型推荐;带 tools 的请求自动退回模板安全的软开关) | | `--textools` | 工具以文本形式随 prompt 下发,而非原生 tool_calls | @@ -160,7 +161,9 @@ evalharness viz show report.json --style errors # 失败样本下钻 | `mock` / `mock-boxed` / `mock-fc` | 离线适配器(管线自检:回声 / 回放金标 / 回放工具调用) | | `!nothink` `!perf` `!textools` 后缀 | spec 内联开关(与 CLI 参数等价) | -API key 按端点从环境变量读取(`OPENAI_API_KEY`、`ANTHROPIC_API_KEY` …)。 +API key(含 judge)按端点域名自动从环境变量读取:`api.openai.com`→`OPENAI_API_KEY`、 +`anthropic.com`→`ANTHROPIC_API_KEY`、`dashscope`→`DASHSCOPE_API_KEY`、`bigmodel`→`ZAI_API_KEY`; +其余域名回退 `OPENAI_API_KEY`。自建端点无鉴权时无需设置。 ## 内置 benchmark diff --git a/evalharness/cli.py b/evalharness/cli.py index a306740..5a27a44 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -233,8 +233,10 @@ def _compose_judge_spec(args): legacy spec; keep both working like the main model flags.""" judge = getattr(args, 'judge', '') or '' url = getattr(args, 'judge_api_url', '') or '' + provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' + internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) if url and judge and '/' not in judge: - judge = f'openai/{url.rstrip("/")}?{judge}' + judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None @@ -243,8 +245,10 @@ def _compose_judge_spec(args): legacy spec; both keep working, mirroring the main model flags.""" judge = getattr(args, 'judge', '') or '' url = getattr(args, 'judge_api_url', '') or '' + provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat' + internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider) if url and judge and '/' not in judge: - judge = f'openai/{url.rstrip("/")}?{judge}' + judge = f'{internal}/{url.rstrip("/")}?{judge}' return judge or None @@ -520,6 +524,9 @@ def build_parser() -> argparse.ArgumentParser: help='judge model name with --judge-api-url, or full spec') p.add_argument('--judge-api-url', default='', help='judge API base URL when --judge is only the model name') + p.add_argument('--judge-provider', default='openai-chat', + help='judge protocol/provider (default openai-chat; ' + 'openai-pool for multi-endpoint judges)') p.add_argument('--profile', default='', help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short ' 'or any @register_gen_profile name); layers: plugin default < '