#!/usr/bin/env python3 """FP-Fusion strict 执行器 (evalstone 兼容 CLI, v1.2 增加维度A/C). 用法(整合进 EvalHarness 后, 三种等价入口): evalharness fingerprint run --api-url http://localhost:30002/v1 \ --model Qwen3-4B --report-path <...>/reports/fp_fusion.json \ [--reference glm53 | /path/to/ref.json] # 不带 = 自证模式(裁决上限 LIKELY_MATCH) python -m evalharness.fingerprint.run_fp_fusion ... # 参数相同 python evalharness/fingerprint/run_fp_fusion.py ... # 直接执行亦兼容 模式 (--mode): verify : 原行为——分布+身份+元知识融合 (默认, 保持兼容) attribution : 增加家族归因信号(S_fam, 词表+可选LLMmap双路) adversarial : attribution 基础上 + 对抗冒充探针(伪装/挑战/风格模仿) 产出: report-path : 统一 Schema 报告(含 score/num, collect_results 可汇总) report-path 同目录 raw_answers.jsonl : 全部探针原文(人工复核用) """ import argparse import asyncio import json import os import sys import time from pathlib import Path if __package__ in (None, ''): # 直接执行: 以包成员重新导入(相对导入需要包上下文) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from evalharness.fingerprint.run_fp_fusion import main as _pkg_main sys.exit(_pkg_main(sys.argv[1:])) from .battery import (ALL_CELL_DEFS, ALL_TEXT_PROBES, CORE16_CELLS, TEXT_PRUNED_V7, TEXT_TEMPERATURE) from .engine import (FusionEngine, build_d_normalized, compare_cells, distributions_by_cell, load_reference, split_half_jsd) from .scorer import build_report, load_aliases MODES = ('verify', 'attribution', 'adversarial', 'variant', 'robustness', 'full') def _assemble_probes(mode, impersonate, text_skip=None): """按模式组装文本探针。verify=原 36 条;attribution/adversarial 增加对抗组。 text_skip: 待剔除的文本探针 id 集合(剪枝落地:TEXT_PRUNED_V7)。""" probes = [p for p in ALL_TEXT_PROBES if not (text_skip and p['id'] in text_skip)] if mode == 'verify': return probes if mode in ('adversarial',): from .probes_adv import ALL_ADV_PROBES adv = ALL_ADV_PROBES() if impersonate: # 显式伪装角色:仅跑角色组 + 挑战组 adv = [p for p in adv if p['id'].startswith(('adv_role_', 'adv_challenge_'))] for p in adv: probes.append(p) if mode in ('variant', 'robustness'): from .probes_variant import ALL_VARIANT_PROBES for p in ALL_VARIANT_PROBES(): probes.append(p) # robustness 复用对抗挑战组(观察角度更多)但不注入伪装 if mode == 'robustness': from .probes_adv import ALL_ADV_PROBES for p in ALL_ADV_PROBES(): probes.append(p) return probes def _load_llmmap_tool(tools_root): """可选 LLMmap 辅助归因:加载 60 模板库(离线)。失败返回 None。 模型目录解析顺序:FP_LLMMAP_MODEL_HOME 环境变量(显式指定 pretrained_models 目录,整合进 EvalHarness 后推荐)→ /LLMmap 内置布局 → 包外旧 相对布局 ../model_library/llmmap(fp_fusion 独立部署时期的位置,已随迁移失效)。 """ try: os.environ.setdefault('HF_HUB_OFFLINE', '1') os.environ.setdefault('TRANSFORMERS_OFFLINE', '1') llmmap_root = os.path.join(tools_root, 'LLMmap') sys.path.insert(0, llmmap_root) from LLMmap.inference import load_LLMmap candidates = [] env_home = os.environ.get('FP_LLMMAP_MODEL_HOME') if env_home: candidates.append(env_home) candidates.append(os.path.join(llmmap_root, 'data', 'pretrained_models', 'default')) candidates.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'model_library', 'llmmap', 'pretrained_models', 'default')) model_home = next((c for c in candidates if os.path.isdir(c)), candidates[0]) _, llmmap = load_LLMmap(model_home, device='cpu') return llmmap if getattr(llmmap, 'ready', False) else None except Exception as e: print(f'[fp_fusion] llmmap attribution disabled: {str(e)[:120]}', file=sys.stderr) return None def resolve_reference(value): """--reference 解析:已存在的路径原样返回;短名(如 glm53)在包内 references/ 依次尝试 _fusion_reference.json → _reference.json。 都找不到时原样返回,由 load_reference 给出报错。""" if not value: return value if Path(value).exists(): return value rdir = Path(__file__).resolve().parent / 'references' for cand in (rdir / f'{value}_fusion_reference.json', rdir / f'{value}_reference.json'): if cand.exists(): return str(cand) return value def _run_full(args, report_path, raw_path, extra_body, d_cells, text_skip, reference_info, ref_cells): """模式合并(--mode full):三通道一次采集,五视图离线打分。 Pass 1 clean : 电池全量(D + 文本 + V + 基线),logprobs 仅在 --logprobs 时请求 (DS 后端对 logprobs 参数直接 400,GLM 后端静默忽略且从不返回—— vectron 上该字段无收益纯风险,默认关),prompt_variants=3 全池 轮换(池均=3 条改写;全池轮换=参考协议的均匀边缘分布,variants=2 会漏 1/3 池导致改写敏感 cell 假性 dist_outlier,实测 city:en JSD 0.117→0.558) Pass 2 injected : 注入态全量文本 + ADV(需 --impersonate;缺省则跳过该通道) Pass 3 sweep : 文本层 × 额外温度点(默认 0.0/1.0;0.2 基线点复用 Pass 1) 视图: verify/attribution/variant ← clean;adversarial ← injected(ADV + 注入态 I/K); robustness ← clean+sweep(温度轴)+ clean(语言轴/改写轴) 注: verify 视图带 attribution(s_fam≠0),分数与历史 attribution 报告同口径(上限 1.0)。 """ from .attribution import family_attribution from .probes_adv import adversarial_signal from .probes_variant import variant_signal from .scorer import requested_family, robustness_signal aliases = load_aliases(args.aliases) req_family = requested_family(args.model, aliases) sweep = ([float(x.strip()) for x in args.temperature_sweep.split(',') if x.strip()] if args.temperature_sweep else [0.0, 1.0]) def make_engine(**kw): params = dict(api_url=args.api_url, model=args.model, timeout=args.timeout, d_samples=args.d_samples, baseline_samples=args.baseline_samples, d_concurrency=args.d_concurrency, text_concurrency=args.text_concurrency, text_max_tokens=args.text_max_tokens, extra_body=extra_body, api_key=args.api_key) params.update(kw) return FusionEngine(**params) all_records = [] passes = [] tokens_in = tokens_out = 0 t0 = time.monotonic() # ---- Pass 1: 清洁主采集 ---- # prompt_variants=3 = 全池轮换(所有 cell 池均为 3 条改写): 边缘分布与参考采集协议 # (全池随机)一致且无 RNG; 若用 2 会漏掉 1/3 池, 改写敏感 cell 会被误判 dist_outlier # logprobs 默认关: DS 后端 400 拒绝该参数(实测), vectron-GLM 静默忽略且从不返回 eng = make_engine(logprobs=args.logprobs, prompt_variants=3, d_cells=d_cells) probes = _assemble_probes('variant', None, text_skip) recs = asyncio.run(eng.run(probes)) for r in recs: r['cond'] = 'clean' all_records.extend(recs) tokens_in += eng.tokens_in tokens_out += eng.tokens_out passes.append(('clean', len(recs), sum(1 for r in recs if not r['error']))) baseline_p50 = eng.baseline_p50 # ---- Pass 2: 注入态全量文本 + ADV ---- if args.impersonate: eng2 = make_engine(d_samples=0, baseline_samples=0, system_prompt_override=args.impersonate) probes2 = _assemble_probes('adversarial', args.impersonate, text_skip) recs2 = asyncio.run(eng2.run(probes2)) for r in recs2: r['cond'] = 'injected' all_records.extend(recs2) tokens_in += eng2.tokens_in tokens_out += eng2.tokens_out passes.append(('injected', len(recs2), sum(1 for r in recs2 if not r['error']))) else: print('[fp_fusion] full: 未提供 --impersonate,跳过注入通道(adversarial 视图禁用)') # ---- Pass 3: 扰动采集(温度轴)---- eng3 = make_engine(d_samples=0, baseline_samples=0, temperature_sweep=sweep) probes3 = _assemble_probes('verify', None, text_skip) recs3 = asyncio.run(eng3.run(probes3)) for r in recs3: r['cond'] = 'sweep' all_records.extend(recs3) tokens_in += eng3.tokens_in tokens_out += eng3.tokens_out passes.append(('sweep', len(recs3), sum(1 for r in recs3 if not r['error']))) elapsed = time.monotonic() - t0 with open(raw_path, 'w', encoding='utf-8') as f: for r in all_records: f.write(json.dumps(r, ensure_ascii=False) + '\n') clean = [r for r in all_records if r.get('cond') == 'clean'] injected = [r for r in all_records if r.get('cond') == 'injected'] swept = [r for r in all_records if r.get('cond') == 'sweep'] # ---- verify/attribution 视图(clean)---- d_norm = build_d_normalized(clean) split_half = split_half_jsd(d_norm) if ref_cells: dist_a = distributions_by_cell(d_norm) entries, mean_jsd = compare_cells(dist_a, ref_cells) outliers = [e for e in entries if e['jsd'] > 0.5 and min(e['valid_a'], e['valid_b']) >= 15] if mean_jsd is not None: sh = split_half if split_half and split_half > 0 else 0.02 ratio = mean_jsd / max(sh, 0.02) s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6) if mean_jsd > 0.35: s_val = min(s_val, 0.2) s = {'s_dist': s_val, 'mean_jsd': mean_jsd, 'relative_ratio': round(ratio, 2), 'split_half': split_half, 'comparable_cells': len(entries), 'most_divergent': entries[:5], 'dist_outlier': bool(outliers), 'outlier_cells': [{'cell': o['cell'], 'jsd': round(o['jsd'], 3)} for o in outliers]} else: s = {'s_dist': None, 'mean_jsd': None, 'comparable_cells': 0, 'dist_outlier': False, 'outlier_cells': []} dist_cmp = {**s, 'baseline_p50': baseline_p50} else: dist_cmp = {'mean_jsd': None, 'split_half': split_half, 'baseline_p50': baseline_p50} llmmap_tool = _load_llmmap_tool(args.tools_root) if args.llmmap_attribution else None attribution = family_attribution(clean, aliases=aliases, requested_family=req_family, llmmap_tool=llmmap_tool) # ---- adversarial 视图(注入态记录:ADV + 伪装条件下的 I/K 泄露扫描)---- adversarial = None if injected: adv_records = [r for r in injected if r.get('layer') == 'ADV'] adversarial = adversarial_signal(adv_records, all_records=injected, requested_family=req_family, dist_family=req_family, aliases=aliases, mode='adversarial', impersonate_role=args.impersonate) report = build_report(clean, d_norm, dist_cmp, args.model, reference_info, aliases, {'input': tokens_in, 'output': tokens_out}, elapsed, attribution=attribution, adversarial=adversarial, mode='full') # ---- variant 视图 ---- report['signals']['variant'] = variant_signal( clean, logprobs_enabled=args.logprobs, notes=['merged: graybox via Pass1 --logprobs' if args.logprobs else 'merged: graybox off (vectron 不返回 logprobs; DS 后端 400 拒绝)', 'self-consistency via v_determinism_a/b']) # ---- robustness 视图(温度轴 = clean 基线点 + sweep;语言/改写轴 = clean)---- report['signals']['robustness'] = robustness_signal( clean + swept, temperature_sweep=sorted({TEXT_TEMPERATURE} | set(sweep))) report['passes'] = {name: {'probes': total, 'successful': ok} for name, total, ok in passes} report['logprobs_sampled'] = sum(1 for r in clean if r.get('top_logprobs')) with open(report_path, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) print(f"[fp_fusion] mode=full verdict={report['verdict']} score={report['score']} | " f"gate={report['gate']['quality']} " f"({report['gate']['successful_probes']}/{report['gate']['total_probes']}) | " f"passes=" + " ".join(f"{n}:{ok}/{t}" for n, t, ok in passes) + f" | elapsed={elapsed:.0f}s") fam = report['signals'].get('family') or {} print(f"[fp_fusion] family: top1={fam.get('top1_family')} " f"conf={fam.get('confidence')} s_fam={fam.get('s_fam')}") if adversarial: print(f"[fp_fusion] adv: impersonation={adversarial.get('impersonation_flag')} " f"role_yield={adversarial.get('role_yield')} " f"conflict={adversarial.get('claimed_behavior_conflict')}") print(f"[fp_fusion] report: {report_path}\n[fp_fusion] raw: {raw_path}") return report def main(argv=None): parser = argparse.ArgumentParser( prog='fp_fusion', description='FP-Fusion strict benchmark (v1.2, dims A/C)') parser.add_argument('--api-url', required=True) parser.add_argument('--model', required=True) parser.add_argument('--report-path', required=True) parser.add_argument('--timeout', type=int, default=120) parser.add_argument('--tools-root', default=os.environ.get('FP_TOOLS_ROOT', '/data1/xii')) parser.add_argument('--reference', default=None, help='detector-schema reference JSON; omit = self mode') parser.add_argument('--aliases', default=None, help='family_aliases.json override') parser.add_argument('--d-samples', type=int, default=20) parser.add_argument('--baseline-samples', type=int, default=20) parser.add_argument('--text-limit', type=int, default=0, help='>0 只跑前 N 条文本探针(冒烟)') parser.add_argument('--d-concurrency', type=int, default=4) parser.add_argument('--text-concurrency', type=int, default=3) parser.add_argument('--text-max-tokens', type=int, default=256) # ---- 维度 A/C ---- parser.add_argument('--mode', choices=MODES, default='verify', help='verify=原行为 | attribution=+家族归因 | adversarial=+对抗冒充 | ' 'full=模式合并:三通道一次采集(clean/injected/sweep),五视图离线打分 ' '(需 --impersonate 启用对抗视图)') parser.add_argument('--impersonate', default=None, help='对抗模式:注入伪装角色 system prompt(如 "You are GPT-4o...")') parser.add_argument('--logprobs', action='store_true', default=False, help='灰盒预留:采集首个 token 的 top-logprobs(一期不评分)') parser.add_argument('--llmmap-attribution', action='store_true', default=False, help='启用 LLMmap 嵌入辅助归因(需 torch+e5)') parser.add_argument('--extra-body', default=None, help='附加请求体字段 JSON,如 {"thinking":{"type":"disabled"}}') # ---- 维度 D ---- parser.add_argument('--temperature-sweep', default=None, help='文本层温度扫描,逗号分隔如 "0.0,0.7,1.0"(robustness 用)') parser.add_argument('--prompt-variants', type=int, default=0, help='>0 时 D 层每 cell 轮换前 N 个 paraphrase(改写轴)') # ---- 剪枝落地(2026-09-07 分析结论;默认关闭,保持旧行为)---- parser.add_argument('--cells', default='all', help="'all'=全 26 cell(默认) | 'core16'=剪枝定稿集 | 逗号分隔 cell 清单") parser.add_argument('--text-skip', default='none', help="'none'=全 36 条(默认) | 'pruned7'=剪枝定稿 7 条 | 逗号分隔探针 id") parser.add_argument('--api-key', default=None, help='Bearer API key(vectron 等需要鉴权;本地端点可省略)') args = parser.parse_args(argv) report_path = Path(args.report_path).resolve() report_path.parent.mkdir(parents=True, exist_ok=True) raw_path = report_path.parent / 'raw_answers.jsonl' extra_body = None if args.extra_body: try: extra_body = json.loads(args.extra_body) except json.JSONDecodeError as e: print(f'ERROR: --extra-body 不是合法 JSON: {e}', file=sys.stderr) sys.exit(1) temp_sweep = None if args.temperature_sweep: temp_sweep = [float(x.strip()) for x in args.temperature_sweep.split(',') if x.strip()] # ---- 剪枝参数解析(--cells / --text-skip)---- if args.cells == 'all': d_cells = None elif args.cells == 'core16': d_cells = set(CORE16_CELLS) else: d_cells = {x.strip() for x in args.cells.split(',') if x.strip()} universe = {f"{c['id']}:{l}" for c in ALL_CELL_DEFS for l in ('en', 'zh')} bad = d_cells - universe if bad: print(f'ERROR: --cells 含未知 cell: {sorted(bad)}', file=sys.stderr) sys.exit(1) if args.text_skip == 'none': text_skip = set() elif args.text_skip == 'pruned7': text_skip = set(TEXT_PRUNED_V7) else: text_skip = {x.strip() for x in args.text_skip.split(',') if x.strip()} known = {p['id'] for p in ALL_TEXT_PROBES} bad = text_skip - known if bad: print(f'ERROR: --text-skip 含未知探针: {sorted(bad)}', file=sys.stderr) sys.exit(1) reference_info, ref_cells = None, None if args.reference: ref = load_reference(resolve_reference(args.reference)) reference_info, ref_cells = ref['model'], ref['cells'] # ---- 模式合并:三通道一次采集,五视图打分 ---- if args.mode == 'full': _run_full(args, report_path, raw_path, extra_body, d_cells, text_skip, reference_info, ref_cells) return # variant 模式必须开 logprobs(灰盒信号) logprobs = args.logprobs or (args.mode == 'variant') engine = FusionEngine(api_url=args.api_url, model=args.model, timeout=args.timeout, d_samples=args.d_samples, baseline_samples=args.baseline_samples, text_limit=args.text_limit, d_concurrency=args.d_concurrency, text_concurrency=args.text_concurrency, text_max_tokens=args.text_max_tokens, extra_body=extra_body, system_prompt_override=args.impersonate, logprobs=logprobs, temperature_sweep=temp_sweep, prompt_variants=args.prompt_variants, d_cells=d_cells, api_key=args.api_key) probes = _assemble_probes(args.mode, args.impersonate, text_skip) n_cells = 26 if d_cells is None else len(d_cells) print(f"[fp_fusion] battery: cells={n_cells}/26 (D={n_cells * args.d_samples} req) " f"text probes={len(probes)} baseline={args.baseline_samples}") t0 = time.monotonic() records = asyncio.run(engine.run(probes)) elapsed = time.monotonic() - t0 with open(raw_path, 'w', encoding='utf-8') as f: for r in records: f.write(json.dumps(r, ensure_ascii=False) + '\n') d_norm = build_d_normalized(records) split_half = split_half_jsd(d_norm) if ref_cells: dist_a = distributions_by_cell(d_norm) entries, mean_jsd = compare_cells(dist_a, ref_cells) # v1.1 dist_outlier 规则: 单 cell 极端分化(双方≥15有效且JSD>0.5) outliers = [e for e in entries if e['jsd'] > 0.5 and min(e['valid_a'], e['valid_b']) >= 15] s = dict() if mean_jsd is not None: sh = split_half if split_half and split_half > 0 else 0.02 ratio = mean_jsd / max(sh, 0.02) s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6) if mean_jsd > 0.35: s_val = min(s_val, 0.2) s = {'s_dist': s_val, 'mean_jsd': mean_jsd, 'relative_ratio': round(ratio, 2), 'split_half': split_half, 'comparable_cells': len(entries), 'most_divergent': entries[:5], 'dist_outlier': bool(outliers), 'outlier_cells': [{'cell': o['cell'], 'jsd': round(o['jsd'], 3)} for o in outliers]} else: s = {'s_dist': None, 'mean_jsd': None, 'comparable_cells': 0, 'dist_outlier': False, 'outlier_cells': [], 'note': 'no comparable cells (valid samples too few)'} dist_cmp = {**s, 'baseline_p50': engine.baseline_p50} else: dist_cmp = {'mean_jsd': None, 'split_half': split_half, 'baseline_p50': engine.baseline_p50} aliases = load_aliases(args.aliases) req_family = None if args.mode != 'verify': from .scorer import requested_family req_family = requested_family(args.model, aliases) # ---- 维度 A:家族归因 ---- attribution = None if args.mode != 'verify': from .attribution import family_attribution llmmap_tool = _load_llmmap_tool(args.tools_root) if args.llmmap_attribution else None attribution = family_attribution(records, aliases=aliases, requested_family=req_family, llmmap_tool=llmmap_tool) # ---- 维度 C:对抗信号 ---- adversarial = None if args.mode == 'adversarial': from .probes_adv import adversarial_signal adv_records = [r for r in records if r.get('layer') == 'ADV'] dist_family = req_family adversarial = adversarial_signal(adv_records, all_records=records, requested_family=req_family, dist_family=dist_family, aliases=aliases, mode=args.mode, impersonate_role=args.impersonate) report = build_report(records, d_norm, dist_cmp, args.model, reference_info, aliases, {'input': engine.tokens_in, 'output': engine.tokens_out}, elapsed, attribution=attribution, adversarial=adversarial, mode=args.mode) # ---- 维度 B:变体区分信号 ---- if args.mode == 'variant': from .probes_variant import variant_signal report['signals']['variant'] = variant_signal( records, logprobs_enabled=logprobs, notes=['graybox via --logprobs', 'self-consistency via v_determinism_a/b']) # ---- 维度 D:鲁棒性信号 ---- if args.mode == 'robustness': from .scorer import robustness_signal report['signals']['robustness'] = robustness_signal( records, temperature_sweep=temp_sweep) if logprobs: report['logprobs_sampled'] = sum(1 for r in records if r.get('top_logprobs')) with open(report_path, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) print(f"[fp_fusion] mode={report['mode']} verdict={report['verdict']} " f"score={report['score']} | gate={report['gate']['quality']} " f"({report['gate']['successful_probes']}/{report['gate']['total_probes']}) | " f"meanJSD={report['signals']['dist'].get('mean_jsd')} | " f"latency p50={engine.baseline_p50}ms elapsed={elapsed:.0f}s") if attribution: print(f"[fp_fusion] family: top1={attribution.get('top1_family')} " f"conf={attribution.get('confidence')} s_fam={attribution.get('s_fam')}") if adversarial: print(f"[fp_fusion] adv: impersonation={adversarial.get('impersonation_flag')} " f"role_yield={adversarial.get('role_yield')} " f"conflict={adversarial.get('claimed_behavior_conflict')}") print(f"[fp_fusion] report: {report_path}\n[fp_fusion] raw: {raw_path}") return 0 if __name__ == '__main__': sys.exit(main(sys.argv[1:]))