Add --api-key/--judge-api-key: explicit keys override env resolution, never serialized
Explicit key flows only into request headers (adapter attribute / pool member), so it cannot leak into the spec string, EvalReport, or logs -- verified by scanning a report produced with a sentinel key. Two-key setups run twice with different --api-key, or use per-host env vars. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora
parent
ad1cd18f04
commit
c78b0d6f0f
@ -69,7 +69,9 @@ evalharness eval run gsm8k \
|
||||
| `--model NAME` | 服务端模型名（配合 `--api-url`）；或直接给完整 spec：`openai/http://h:8000/v1?qwen3-8b` |
|
||||
| `--provider {openai-chat,openai-pool}` | 协议/提供方，默认 `openai-chat`；端点池用 `openai-pool` |
|
||||
| `--judge NAME` + `--judge-api-url URL` | LLM-judge 模型（hle / simple_qa / imo 需要）；也可单给完整 spec `--judge openai/http://...?m` |
|
||||
| `--api-key KEY` | 显式指定主模型端点的 key（优先于环境变量推断；只进请求头，不写入 spec/报告） |
|
||||
| `--judge-provider {openai-chat,openai-pool}` | judge 协议；多 judge 端点负载均衡用 `openai-pool` |
|
||||
| `--judge-api-key KEY` | 显式指定 judge 端点的 key |
|
||||
| `--profile NAME` | 命名生成参数集（内置 `dp4-nothink`、`qwen3-es-parity`、`t1-short`，或任意 `@register_gen_profile` 名）；优先级：插件默认 < profile 默认 < profile 单 bench 覆盖 < 显式参数 |
|
||||
| `--disable-thinking` | 发送 `enable_thinking=false`（Qwen3 类思考模型推荐；带 tools 的请求自动退回模板安全的软开关） |
|
||||
| `--textools` | 工具以文本形式随 prompt 下发，而非原生 tool_calls |
|
||||
@ -164,6 +166,8 @@ evalharness viz show report.json --style errors # 失败样本下钻
|
||||
API key（含 judge）按端点域名自动从环境变量读取：`api.openai.com`→`OPENAI_API_KEY`、
|
||||
`anthropic.com`→`ANTHROPIC_API_KEY`、`dashscope`→`DASHSCOPE_API_KEY`、`bigmodel`→`ZAI_API_KEY`；
|
||||
其余域名回退 `OPENAI_API_KEY`。自建端点无鉴权时无需设置。
|
||||
同一服务有多个 key 时用 `--api-key` 显式指定（跑两次各用一个 key 即可对比；key 只进请求头，
|
||||
不会出现在 spec、报告或日志里）。
|
||||
|
||||
## 内置 benchmark
|
||||
|
||||
|
||||
@ -308,6 +308,8 @@ def _cmd_eval_run(args) -> int:
|
||||
limit_per_task=args.limit_per_task,
|
||||
checkpoint=args.resume,
|
||||
judge_spec=_compose_judge_spec(args), env=args.env,
|
||||
api_key=getattr(args, 'api_key', ''),
|
||||
judge_api_key=getattr(args, 'judge_api_key', ''),
|
||||
gen_profile=getattr(args, 'profile', ''),
|
||||
progress_reporter=progress_reporter,
|
||||
status_callback=status_callback))
|
||||
@ -524,6 +526,11 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help='judge model name with --judge-api-url, or full spec')
|
||||
p.add_argument('--judge-api-url', default='',
|
||||
help='judge API base URL when --judge is only the model name')
|
||||
p.add_argument('--api-key', default='',
|
||||
help='explicit API key for the model endpoint (overrides '
|
||||
'env-based resolution; never written into reports)')
|
||||
p.add_argument('--judge-api-key', default='',
|
||||
help='explicit API key for the judge endpoint')
|
||||
p.add_argument('--judge-provider', default='openai-chat',
|
||||
help='judge protocol/provider (default openai-chat; '
|
||||
'openai-pool for multi-endpoint judges)')
|
||||
|
||||
@ -245,8 +245,15 @@ class AdaptiveGate:
|
||||
**{f'gate_{k}': v for k, v in self.stats.items()}}
|
||||
|
||||
|
||||
def pooled(specs: List[str]) -> PooledAdapter:
|
||||
"""['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter."""
|
||||
def pooled(specs: List[str], api_key: str = '') -> PooledAdapter:
|
||||
"""['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter.
|
||||
|
||||
api_key: explicit key applied to EVERY member (two-key setups should
|
||||
build two pools, or use env resolution per host)."""
|
||||
from .adapter import resolve_adapter
|
||||
|
||||
return PooledAdapter([resolve_adapter(s) for s in specs])
|
||||
members = [resolve_adapter(s) for s in specs]
|
||||
if api_key:
|
||||
for m in members:
|
||||
m.api_key = api_key
|
||||
return PooledAdapter(members)
|
||||
|
||||
@ -502,6 +502,8 @@ async def run_eval(
|
||||
concurrency: int = 32,
|
||||
limit: Optional[int] = None,
|
||||
gen_kwargs: Optional[Dict[str, Any]] = None,
|
||||
api_key: str = '',
|
||||
judge_api_key: str = '',
|
||||
judge_spec: Optional[str] = None,
|
||||
judge: Optional[Any] = None,
|
||||
progress: bool = True,
|
||||
@ -535,7 +537,7 @@ async def run_eval(
|
||||
spec = getattr(dataset, 'spec', None)
|
||||
if few_shot_num < 0:
|
||||
few_shot_num = (spec.few_shot_num if spec is not None else 0)
|
||||
adapter = _make_adapter(model_spec)
|
||||
adapter = _make_adapter(model_spec, api_key=api_key)
|
||||
# reports carry a string model label: pre-built adapter objects need one
|
||||
model_spec = model_spec if isinstance(model_spec, str) \
|
||||
else (getattr(model_spec, 'model', '') or repr(model_spec))
|
||||
@ -650,7 +652,7 @@ async def run_eval(
|
||||
if judge is None and judge_spec:
|
||||
if status_callback:
|
||||
status_callback('loading judge model')
|
||||
judge_adapter = _make_adapter(judge_spec)
|
||||
judge_adapter = _make_adapter(judge_spec, api_key=judge_api_key or api_key)
|
||||
judge = _judge_callable(judge_adapter)
|
||||
|
||||
if status_callback:
|
||||
@ -693,7 +695,7 @@ def _env_user_adapter(spec: str):
|
||||
_ENV_USER_CACHE = {}
|
||||
|
||||
|
||||
def _make_adapter(spec: str) -> ModelAdapter:
|
||||
def _make_adapter(spec: str, api_key: str = '') -> ModelAdapter:
|
||||
"""Model spec forms:
|
||||
- 'mock[:mode]' offline adapter
|
||||
- 'openai-pool/<base-url-template>?model' with {port} placeholder:
|
||||
@ -743,7 +745,7 @@ def _make_adapter(spec: str) -> ModelAdapter:
|
||||
specs.append(f'openai/{u}?{model}')
|
||||
elif seg:
|
||||
specs.append(f'openai/{seg}?{model}')
|
||||
adapter = pooled(specs)
|
||||
adapter = pooled(specs, api_key=api_key) if api_key else pooled(specs)
|
||||
elif re.fullmatch(r'mock[-:](boxed|oracle|fc|tool|echo|const)?', spec):
|
||||
# mock-boxed (preferred) == legacy mock:boxed; bare 'mock' == echo.
|
||||
# NEVER reuse the cached singleton: resolve_adapter memoizes and a
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user