#!/usr/bin/env python3
"""llm-fingerprint-detector benchmark runner（单 token 输出分布指纹）。

两种工作模式：
  1) --reference 提供同协议参考指纹 JSON 时：对被测端点采样一次并与参考比对
     （verify 模式，硬比较）。
  2) 未提供参考时：自一致模式——连续采样两次后互相比对，衡量端点输出分布的
     稳定性（split-half 思路），同时把 splitHalfJsd 记入报告。

由 run.py 以子进程方式调用，只需任意 Python + node（需已 npm run build）：

    <python> run_llm_detector.py --api-url ... --model ... --report-path ...

得分（score ∈ [0,1]）：score = max(0, 1 - meanJSD)，并记录 verdict
    match ≤0.25 < uncertain ≤0.35 < mismatch（论文基线标尺）。
"""

import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path

from common import BENCHMARK_DETECTOR, add_common_args, write_report


def run_cli(cmd: list, timeout: int) -> dict:
    """Run the detector CLI with --json and return parsed stdout JSON."""
    proc = subprocess.run(
        cmd, capture_output=True, text=True, timeout=timeout,
        env={**os.environ, 'LLM_FINGERPRINT_API_KEY': os.environ.get('LLM_FINGERPRINT_API_KEY', 'dummy')},
    )
    if proc.returncode not in (0, 2, 3):  # 2=mismatch 3=uncertain 也是有效结论
        raise RuntimeError(
            f'detector CLI failed (rc={proc.returncode}):\n'
            f'{proc.stdout[-500:]}\n{proc.stderr[-800:]}')
    try:
        return json.loads(proc.stdout)
    except json.JSONDecodeError as e:
        raise RuntimeError(f'cannot parse CLI --json output: {e}\n{proc.stdout[-300:]}')


def base_cmd(args) -> list:
    root = Path(args.tools_root) / 'llm-fingerprint-detector'
    cli = root / 'dist' / 'cli.js'
    if not cli.exists():
        raise FileNotFoundError(f'detector CLI not built: {cli} (run `npm run build` in the repo)')
    return [args.node, str(cli)]


def endpoint_cmd(args) -> list:
    return ['--base-url', args.api_url.rstrip('/'), '--model', args.model,
            '--preset', args.preset, '--timeout', str(args.timeout * 1000),
            '--concurrency', str(args.concurrency), '--json']


def main():
    parser = argparse.ArgumentParser(description='llm-fingerprint-detector benchmark')
    add_common_args(parser)
    parser.add_argument('--tools-root', default='/data1/xii',
                        help='Directory containing the cloned llm-fingerprint-detector repo')
    parser.add_argument('--node', default=os.environ.get('DETECTOR_NODE', 'node'),
                        help='Node executable (default: %(default)s)')
    parser.add_argument('--reference', default=None,
                        help='Same-protocol reference fingerprint JSON; '
                             'omit for self-consistency mode')
    parser.add_argument('--preset', default='standard',
                        choices=['quick', 'standard', 'strict'])
    parser.add_argument('--concurrency', type=int, default=4)
    args = parser.parse_args()

    cmd = base_cmd(args) + endpoint_cmd(args)

    if args.reference:
        # ---- verify 模式：与参考指纹硬比较 ----
        out = run_cli(cmd + ['verify', '--reference', args.reference], timeout=args.timeout * 40)
        mean_jsd = float(out.get('meanJsd', out.get('comparison', {}).get('meanJsd', 1.0)))
        verdict = out.get('verdict', 'insufficient')
        mode = 'reference_verify'
        reference = args.reference
        split_half = None
        cells = out.get('comparison', {}).get('cells') or out.get('cells') or []
    else:
        # ---- 自一致模式：采两次互相比较 ----
        tmp_dir = Path(args.report_path).resolve().parent.parent / 'detector_tmp'
        tmp_dir.mkdir(parents=True, exist_ok=True)
        fa, fb = tmp_dir / 'fp_a.json', tmp_dir / 'fp_b.json'

        run_a = run_cli(cmd + ['fingerprint', '--out', str(fa)], timeout=args.timeout * 40)
        run_b = run_cli(cmd + ['fingerprint', '--out', str(fb)], timeout=args.timeout * 40)
        cmp_out = run_cli(base_cmd(args) + ['compare', str(fa), str(fb), '--json'],
                          timeout=60)

        mean_jsd = float(cmp_out.get('meanJsd', 1.0))
        verdict = cmp_out.get('verdict', 'insufficient')
        mode = 'self_consistency'
        reference = None
        split_half = (run_a.get('run') or {}).get('splitHalfJsd')
        cells = cmp_out.get('cells') or []

    score = max(0.0, min(1.0, 1.0 - mean_jsd))

    write_report(
        args.report_path, BENCHMARK_DETECTOR, score,
        num=len(cells),
        mode=mode,
        verdict=verdict,
        mean_jsd=mean_jsd,
        split_half_jsd=split_half,
        reference=reference,
        preset=args.preset,
        most_divergent=[
            {'cell': c.get('cellId'), 'jsd': c.get('jsd')} for c in cells[:5]
        ],
    )
    print(f"[llm_fingerprint_detector] mode={mode} verdict={verdict} "
          f"meanJSD={mean_jsd:.3f} -> score={score:.3f}")


if __name__ == '__main__':
    main()