ruoxi_sun
4f274c2c32
- fingerprint benchmark: add fp_fusion(26-cell fusion) to collect_results/run.py - run_llmmap.py: default model path to evalstone built-in model_library - add model libraries (llmdetector 11 refs / fp_fusion 8 fusion refs / llmmap templates 60 models incl 8 new: GLM-5.2/5.3, DeepSeek-Flash/Pro/ Flash-0731, Kimi-K3, MiniMax-M2.7, TianGong-Taie) - add fp_fusion engine (battery/engine/scorer) + docs - gitignore: exclude binary model weights and temp backups
123 lines
5.5 KiB
Python
123 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""FP-Fusion strict 执行器 (evalstone 兼容 CLI).
|
|
|
|
用法:
|
|
python run_fp_fusion.py --api-url http://localhost:30002/v1 \
|
|
--model Qwen3-4B --report-path <...>/reports/fp_fusion.json \
|
|
[--reference /path/to/ref.json] # 不带 = 自证模式(裁决上限 LIKELY_MATCH)
|
|
|
|
产出:
|
|
report-path : 统一 Schema 报告(含 score/num, collect_results 可汇总)
|
|
report-path 同目录 raw_answers.jsonl : 全部探针原文(人工复核用)
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from battery import ALL_TEXT_PROBES # noqa: E402
|
|
from engine import (FusionEngine, build_d_normalized, # noqa: E402
|
|
compare_cells, distributions_by_cell, load_reference,
|
|
split_half_jsd)
|
|
from scorer import build_report, load_aliases # noqa: E402
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='FP-Fusion strict benchmark')
|
|
parser.add_argument('--api-url', required=True)
|
|
parser.add_argument('--model', required=True)
|
|
parser.add_argument('--report-path', required=True)
|
|
parser.add_argument('--timeout', type=int, default=120)
|
|
parser.add_argument('--tools-root', default=os.environ.get('FP_TOOLS_ROOT', '/data1/xii'))
|
|
parser.add_argument('--reference', default=None,
|
|
help='detector-schema reference JSON; omit = self mode')
|
|
parser.add_argument('--aliases', default=None, help='family_aliases.json override')
|
|
parser.add_argument('--d-samples', type=int, default=20)
|
|
parser.add_argument('--baseline-samples', type=int, default=20)
|
|
parser.add_argument('--text-limit', type=int, default=0, help='>0 只跑前 N 条文本探针(冒烟)')
|
|
parser.add_argument('--d-concurrency', type=int, default=4)
|
|
parser.add_argument('--text-concurrency', type=int, default=3)
|
|
parser.add_argument('--text-max-tokens', type=int, default=256)
|
|
args = parser.parse_args()
|
|
|
|
report_path = Path(args.report_path).resolve()
|
|
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
raw_path = report_path.parent / 'raw_answers.jsonl'
|
|
|
|
reference_info, ref_cells = None, None
|
|
if args.reference:
|
|
ref = load_reference(args.reference)
|
|
reference_info, ref_cells = ref['model'], ref['cells']
|
|
|
|
engine = FusionEngine(api_url=args.api_url, model=args.model, timeout=args.timeout,
|
|
d_samples=args.d_samples, baseline_samples=args.baseline_samples,
|
|
text_limit=args.text_limit, d_concurrency=args.d_concurrency,
|
|
text_concurrency=args.text_concurrency,
|
|
text_max_tokens=args.text_max_tokens)
|
|
|
|
t0 = time.monotonic()
|
|
records = asyncio.run(engine.run(ALL_TEXT_PROBES))
|
|
elapsed = time.monotonic() - t0
|
|
|
|
with open(raw_path, 'w', encoding='utf-8') as f:
|
|
for r in records:
|
|
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
|
|
|
d_norm = build_d_normalized(records)
|
|
split_half = split_half_jsd(d_norm)
|
|
|
|
if ref_cells:
|
|
dist_a = distributions_by_cell(d_norm)
|
|
entries, mean_jsd = compare_cells(dist_a, ref_cells)
|
|
# v1.1 dist_outlier 规则: 单 cell 极端分化(双方≥15有效且JSD>0.5)
|
|
# → 实锤级信号, 不被均值稀释(兄弟假冒案例: 均值0.27~0.36 但单cell达1.0)
|
|
outliers = [e for e in entries
|
|
if e['jsd'] > 0.5 and min(e['valid_a'], e['valid_b']) >= 15]
|
|
s = dict()
|
|
if mean_jsd is not None:
|
|
sh = split_half if split_half and split_half > 0 else 0.02
|
|
ratio = mean_jsd / max(sh, 0.02)
|
|
s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6)
|
|
if mean_jsd > 0.35:
|
|
s_val = min(s_val, 0.2)
|
|
s = {'s_dist': s_val, 'mean_jsd': mean_jsd,
|
|
'relative_ratio': round(ratio, 2),
|
|
'split_half': split_half,
|
|
'comparable_cells': len(entries),
|
|
'most_divergent': entries[:5],
|
|
'dist_outlier': bool(outliers),
|
|
'outlier_cells': [{'cell': o['cell'], 'jsd': round(o['jsd'], 3)}
|
|
for o in outliers]}
|
|
else:
|
|
s = {'s_dist': None, 'mean_jsd': None, 'comparable_cells': 0,
|
|
'dist_outlier': False, 'outlier_cells': [],
|
|
'note': 'no comparable cells (valid samples too few)'}
|
|
dist_cmp = {**s, 'baseline_p50': engine.baseline_p50}
|
|
else:
|
|
dist_cmp = {'mean_jsd': None, 'split_half': split_half,
|
|
'baseline_p50': engine.baseline_p50}
|
|
|
|
aliases = load_aliases(args.aliases)
|
|
report = build_report(records, d_norm, dist_cmp, args.model, reference_info,
|
|
aliases, {'input': engine.tokens_in, 'output': engine.tokens_out},
|
|
elapsed)
|
|
with open(report_path, 'w', encoding='utf-8') as f:
|
|
json.dump(report, f, ensure_ascii=False, indent=2)
|
|
|
|
print(f"[fp_fusion] mode={report['mode']} verdict={report['verdict']} "
|
|
f"score={report['score']} | gate={report['gate']['quality']} "
|
|
f"({report['gate']['successful_probes']}/{report['gate']['total_probes']}) | "
|
|
f"meanJSD={report['signals']['dist'].get('mean_jsd')} | "
|
|
f"latency p50={engine.baseline_p50}ms elapsed={elapsed:.0f}s")
|
|
print(f"[fp_fusion] report: {report_path}\n[fp_fusion] raw: {raw_path}")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|