124 lines
5.0 KiB
Python
124 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
||
"""llm-fingerprint-detector benchmark runner（单 token 输出分布指纹）。
|
||
|
||
两种工作模式：
|
||
1) --reference 提供同协议参考指纹 JSON 时：对被测端点采样一次并与参考比对
|
||
（verify 模式，硬比较）。
|
||
2) 未提供参考时：自一致模式——连续采样两次后互相比对，衡量端点输出分布的
|
||
稳定性（split-half 思路），同时把 splitHalfJsd 记入报告。
|
||
|
||
由 run.py 以子进程方式调用，只需任意 Python + node（需已 npm run build）：
|
||
|
||
<python> run_llm_detector.py --api-url ... --model ... --report-path ...
|
||
|
||
得分（score ∈ [0,1]）：score = max(0, 1 - meanJSD)，并记录 verdict
|
||
match ≤0.25 < uncertain ≤0.35 < mismatch（论文基线标尺）。
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
from pathlib import Path
|
||
|
||
from common import BENCHMARK_DETECTOR, add_common_args, write_report
|
||
|
||
|
||
def run_cli(cmd: list, timeout: int) -> dict:
|
||
"""Run the detector CLI with --json and return parsed stdout JSON."""
|
||
proc = subprocess.run(
|
||
cmd, capture_output=True, text=True, timeout=timeout,
|
||
env={**os.environ, 'LLM_FINGERPRINT_API_KEY': os.environ.get('LLM_FINGERPRINT_API_KEY', 'dummy')},
|
||
)
|
||
if proc.returncode not in (0, 2, 3): # 2=mismatch 3=uncertain 也是有效结论
|
||
raise RuntimeError(
|
||
f'detector CLI failed (rc={proc.returncode}):\n'
|
||
f'{proc.stdout[-500:]}\n{proc.stderr[-800:]}')
|
||
try:
|
||
return json.loads(proc.stdout)
|
||
except json.JSONDecodeError as e:
|
||
raise RuntimeError(f'cannot parse CLI --json output: {e}\n{proc.stdout[-300:]}')
|
||
|
||
|
||
def base_cmd(args) -> list:
|
||
root = Path(args.tools_root) / 'llm-fingerprint-detector'
|
||
cli = root / 'dist' / 'cli.js'
|
||
if not cli.exists():
|
||
raise FileNotFoundError(f'detector CLI not built: {cli} (run `npm run build` in the repo)')
|
||
return [args.node, str(cli)]
|
||
|
||
|
||
def endpoint_cmd(args) -> list:
|
||
return ['--base-url', args.api_url.rstrip('/'), '--model', args.model,
|
||
'--preset', args.preset, '--timeout', str(args.timeout * 1000),
|
||
'--concurrency', str(args.concurrency), '--json']
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='llm-fingerprint-detector benchmark')
|
||
add_common_args(parser)
|
||
parser.add_argument('--tools-root', default='/data1/xii',
|
||
help='Directory containing the cloned llm-fingerprint-detector repo')
|
||
parser.add_argument('--node', default=os.environ.get('DETECTOR_NODE', 'node'),
|
||
help='Node executable (default: %(default)s)')
|
||
parser.add_argument('--reference', default=None,
|
||
help='Same-protocol reference fingerprint JSON; '
|
||
'omit for self-consistency mode')
|
||
parser.add_argument('--preset', default='standard',
|
||
choices=['quick', 'standard', 'strict'])
|
||
parser.add_argument('--concurrency', type=int, default=4)
|
||
args = parser.parse_args()
|
||
|
||
cmd = base_cmd(args) + endpoint_cmd(args)
|
||
|
||
if args.reference:
|
||
# ---- verify 模式：与参考指纹硬比较 ----
|
||
out = run_cli(cmd + ['verify', '--reference', args.reference], timeout=args.timeout * 40)
|
||
mean_jsd = float(out.get('meanJsd', out.get('comparison', {}).get('meanJsd', 1.0)))
|
||
verdict = out.get('verdict', 'insufficient')
|
||
mode = 'reference_verify'
|
||
reference = args.reference
|
||
split_half = None
|
||
cells = out.get('comparison', {}).get('cells') or out.get('cells') or []
|
||
else:
|
||
# ---- 自一致模式：采两次互相比较 ----
|
||
tmp_dir = Path(args.report_path).resolve().parent.parent / 'detector_tmp'
|
||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||
fa, fb = tmp_dir / 'fp_a.json', tmp_dir / 'fp_b.json'
|
||
|
||
run_a = run_cli(cmd + ['fingerprint', '--out', str(fa)], timeout=args.timeout * 40)
|
||
run_b = run_cli(cmd + ['fingerprint', '--out', str(fb)], timeout=args.timeout * 40)
|
||
cmp_out = run_cli(base_cmd(args) + ['compare', str(fa), str(fb), '--json'],
|
||
timeout=60)
|
||
|
||
mean_jsd = float(cmp_out.get('meanJsd', 1.0))
|
||
verdict = cmp_out.get('verdict', 'insufficient')
|
||
mode = 'self_consistency'
|
||
reference = None
|
||
split_half = (run_a.get('run') or {}).get('splitHalfJsd')
|
||
cells = cmp_out.get('cells') or []
|
||
|
||
score = max(0.0, min(1.0, 1.0 - mean_jsd))
|
||
|
||
write_report(
|
||
args.report_path, BENCHMARK_DETECTOR, score,
|
||
num=len(cells),
|
||
mode=mode,
|
||
verdict=verdict,
|
||
mean_jsd=mean_jsd,
|
||
split_half_jsd=split_half,
|
||
reference=reference,
|
||
preset=args.preset,
|
||
most_divergent=[
|
||
{'cell': c.get('cellId'), 'jsd': c.get('jsd')} for c in cells[:5]
|
||
],
|
||
)
|
||
print(f"[llm_fingerprint_detector] mode={mode} verdict={verdict} "
|
||
f"meanJSD={mean_jsd:.3f} -> score={score:.3f}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|