370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
515 lines
25 KiB
Python
515 lines
25 KiB
Python
#!/usr/bin/env python3
|
||
"""FP-Fusion strict 执行器 (evalstone 兼容 CLI, v1.2 增加维度A/C).
|
||
|
||
用法(整合进 EvalHarness 后, 三种等价入口):
|
||
evalharness fingerprint run --api-url http://localhost:30002/v1 \
|
||
--model Qwen3-4B --report-path <...>/reports/fp_fusion.json \
|
||
[--reference glm53 | /path/to/ref.json] # 不带 = 自证模式(裁决上限 LIKELY_MATCH)
|
||
python -m evalharness.fingerprint.run_fp_fusion ... # 参数相同
|
||
python evalharness/fingerprint/run_fp_fusion.py ... # 直接执行亦兼容
|
||
|
||
模式 (--mode):
|
||
verify : 原行为——分布+身份+元知识融合 (默认, 保持兼容)
|
||
attribution : 增加家族归因信号(S_fam, 词表+可选LLMmap双路)
|
||
adversarial : attribution 基础上 + 对抗冒充探针(伪装/挑战/风格模仿)
|
||
|
||
产出:
|
||
report-path : 统一 Schema 报告(含 score/num, collect_results 可汇总)
|
||
report-path 同目录 raw_answers.jsonl : 全部探针原文(人工复核用)
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
if __package__ in (None, ''): # 直接执行: 以包成员重新导入(相对导入需要包上下文)
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from evalharness.fingerprint.run_fp_fusion import main as _pkg_main
|
||
sys.exit(_pkg_main(sys.argv[1:]))
|
||
|
||
from .battery import (ALL_CELL_DEFS, ALL_TEXT_PROBES,
|
||
CORE16_CELLS, TEXT_PRUNED_V7,
|
||
TEXT_TEMPERATURE)
|
||
from .engine import (FusionEngine, build_d_normalized,
|
||
compare_cells, distributions_by_cell, load_reference,
|
||
split_half_jsd)
|
||
from .scorer import build_report, load_aliases
|
||
|
||
MODES = ('verify', 'attribution', 'adversarial', 'variant', 'robustness', 'full')
|
||
|
||
|
||
def _assemble_probes(mode, impersonate, text_skip=None):
|
||
"""按模式组装文本探针。verify=原 36 条；attribution/adversarial 增加对抗组。
|
||
text_skip: 待剔除的文本探针 id 集合（剪枝落地：TEXT_PRUNED_V7）。"""
|
||
probes = [p for p in ALL_TEXT_PROBES
|
||
if not (text_skip and p['id'] in text_skip)]
|
||
if mode == 'verify':
|
||
return probes
|
||
if mode in ('adversarial',):
|
||
from .probes_adv import ALL_ADV_PROBES
|
||
adv = ALL_ADV_PROBES()
|
||
if impersonate:
|
||
# 显式伪装角色：仅跑角色组 + 挑战组
|
||
adv = [p for p in adv if p['id'].startswith(('adv_role_', 'adv_challenge_'))]
|
||
for p in adv:
|
||
probes.append(p)
|
||
if mode in ('variant', 'robustness'):
|
||
from .probes_variant import ALL_VARIANT_PROBES
|
||
for p in ALL_VARIANT_PROBES():
|
||
probes.append(p)
|
||
# robustness 复用对抗挑战组（观察角度更多）但不注入伪装
|
||
if mode == 'robustness':
|
||
from .probes_adv import ALL_ADV_PROBES
|
||
for p in ALL_ADV_PROBES():
|
||
probes.append(p)
|
||
return probes
|
||
|
||
|
||
def _load_llmmap_tool(tools_root):
|
||
"""可选 LLMmap 辅助归因：加载 60 模板库（离线）。失败返回 None。
|
||
|
||
模型目录解析顺序：FP_LLMMAP_MODEL_HOME 环境变量（显式指定 pretrained_models
|
||
目录，整合进 EvalHarness 后推荐）→ <tools-root>/LLMmap 内置布局 → 包外旧
|
||
相对布局 ../model_library/llmmap（fp_fusion 独立部署时期的位置，已随迁移失效）。
|
||
"""
|
||
try:
|
||
os.environ.setdefault('HF_HUB_OFFLINE', '1')
|
||
os.environ.setdefault('TRANSFORMERS_OFFLINE', '1')
|
||
llmmap_root = os.path.join(tools_root, 'LLMmap')
|
||
sys.path.insert(0, llmmap_root)
|
||
from LLMmap.inference import load_LLMmap
|
||
candidates = []
|
||
env_home = os.environ.get('FP_LLMMAP_MODEL_HOME')
|
||
if env_home:
|
||
candidates.append(env_home)
|
||
candidates.append(os.path.join(llmmap_root, 'data',
|
||
'pretrained_models', 'default'))
|
||
candidates.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||
'..', 'model_library', 'llmmap',
|
||
'pretrained_models', 'default'))
|
||
model_home = next((c for c in candidates if os.path.isdir(c)),
|
||
candidates[0])
|
||
_, llmmap = load_LLMmap(model_home, device='cpu')
|
||
return llmmap if getattr(llmmap, 'ready', False) else None
|
||
except Exception as e:
|
||
print(f'[fp_fusion] llmmap attribution disabled: {str(e)[:120]}', file=sys.stderr)
|
||
return None
|
||
|
||
|
||
def resolve_reference(value):
|
||
"""--reference 解析：已存在的路径原样返回；短名（如 glm53）在包内
|
||
references/ 依次尝试 <name>_fusion_reference.json → <name>_reference.json。
|
||
都找不到时原样返回，由 load_reference 给出报错。"""
|
||
if not value:
|
||
return value
|
||
if Path(value).exists():
|
||
return value
|
||
rdir = Path(__file__).resolve().parent / 'references'
|
||
for cand in (rdir / f'{value}_fusion_reference.json',
|
||
rdir / f'{value}_reference.json'):
|
||
if cand.exists():
|
||
return str(cand)
|
||
return value
|
||
|
||
|
||
def _run_full(args, report_path, raw_path, extra_body, d_cells, text_skip,
|
||
reference_info, ref_cells):
|
||
"""模式合并（--mode full）：三通道一次采集，五视图离线打分。
|
||
|
||
Pass 1 clean : 电池全量（D + 文本 + V + 基线），logprobs 仅在 --logprobs 时请求
|
||
（DS 后端对 logprobs 参数直接 400，GLM 后端静默忽略且从不返回——
|
||
vectron 上该字段无收益纯风险，默认关），prompt_variants=3 全池
|
||
轮换（池均=3 条改写；全池轮换=参考协议的均匀边缘分布，variants=2 会漏
|
||
1/3 池导致改写敏感 cell 假性 dist_outlier，实测 city:en JSD 0.117→0.558）
|
||
Pass 2 injected : 注入态全量文本 + ADV（需 --impersonate；缺省则跳过该通道）
|
||
Pass 3 sweep : 文本层 × 额外温度点（默认 0.0/1.0；0.2 基线点复用 Pass 1）
|
||
视图: verify/attribution/variant ← clean；adversarial ← injected(ADV + 注入态 I/K)；
|
||
robustness ← clean+sweep（温度轴）+ clean（语言轴/改写轴）
|
||
注: verify 视图带 attribution（s_fam≠0），分数与历史 attribution 报告同口径（上限 1.0）。
|
||
"""
|
||
from .attribution import family_attribution
|
||
from .probes_adv import adversarial_signal
|
||
from .probes_variant import variant_signal
|
||
from .scorer import requested_family, robustness_signal
|
||
|
||
aliases = load_aliases(args.aliases)
|
||
req_family = requested_family(args.model, aliases)
|
||
sweep = ([float(x.strip()) for x in args.temperature_sweep.split(',') if x.strip()]
|
||
if args.temperature_sweep else [0.0, 1.0])
|
||
|
||
def make_engine(**kw):
|
||
params = dict(api_url=args.api_url, model=args.model, timeout=args.timeout,
|
||
d_samples=args.d_samples, baseline_samples=args.baseline_samples,
|
||
d_concurrency=args.d_concurrency,
|
||
text_concurrency=args.text_concurrency,
|
||
text_max_tokens=args.text_max_tokens, extra_body=extra_body,
|
||
api_key=args.api_key)
|
||
params.update(kw)
|
||
return FusionEngine(**params)
|
||
|
||
all_records = []
|
||
passes = []
|
||
tokens_in = tokens_out = 0
|
||
t0 = time.monotonic()
|
||
|
||
# ---- Pass 1: 清洁主采集 ----
|
||
# prompt_variants=3 = 全池轮换(所有 cell 池均为 3 条改写): 边缘分布与参考采集协议
|
||
# (全池随机)一致且无 RNG; 若用 2 会漏掉 1/3 池, 改写敏感 cell 会被误判 dist_outlier
|
||
# logprobs 默认关: DS 后端 400 拒绝该参数(实测), vectron-GLM 静默忽略且从不返回
|
||
eng = make_engine(logprobs=args.logprobs, prompt_variants=3, d_cells=d_cells)
|
||
probes = _assemble_probes('variant', None, text_skip)
|
||
recs = asyncio.run(eng.run(probes))
|
||
for r in recs:
|
||
r['cond'] = 'clean'
|
||
all_records.extend(recs)
|
||
tokens_in += eng.tokens_in
|
||
tokens_out += eng.tokens_out
|
||
passes.append(('clean', len(recs), sum(1 for r in recs if not r['error'])))
|
||
baseline_p50 = eng.baseline_p50
|
||
|
||
# ---- Pass 2: 注入态全量文本 + ADV ----
|
||
if args.impersonate:
|
||
eng2 = make_engine(d_samples=0, baseline_samples=0,
|
||
system_prompt_override=args.impersonate)
|
||
probes2 = _assemble_probes('adversarial', args.impersonate, text_skip)
|
||
recs2 = asyncio.run(eng2.run(probes2))
|
||
for r in recs2:
|
||
r['cond'] = 'injected'
|
||
all_records.extend(recs2)
|
||
tokens_in += eng2.tokens_in
|
||
tokens_out += eng2.tokens_out
|
||
passes.append(('injected', len(recs2),
|
||
sum(1 for r in recs2 if not r['error'])))
|
||
else:
|
||
print('[fp_fusion] full: 未提供 --impersonate，跳过注入通道（adversarial 视图禁用）')
|
||
|
||
# ---- Pass 3: 扰动采集（温度轴）----
|
||
eng3 = make_engine(d_samples=0, baseline_samples=0, temperature_sweep=sweep)
|
||
probes3 = _assemble_probes('verify', None, text_skip)
|
||
recs3 = asyncio.run(eng3.run(probes3))
|
||
for r in recs3:
|
||
r['cond'] = 'sweep'
|
||
all_records.extend(recs3)
|
||
tokens_in += eng3.tokens_in
|
||
tokens_out += eng3.tokens_out
|
||
passes.append(('sweep', len(recs3), sum(1 for r in recs3 if not r['error'])))
|
||
elapsed = time.monotonic() - t0
|
||
|
||
with open(raw_path, 'w', encoding='utf-8') as f:
|
||
for r in all_records:
|
||
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
||
|
||
clean = [r for r in all_records if r.get('cond') == 'clean']
|
||
injected = [r for r in all_records if r.get('cond') == 'injected']
|
||
swept = [r for r in all_records if r.get('cond') == 'sweep']
|
||
|
||
# ---- verify/attribution 视图（clean）----
|
||
d_norm = build_d_normalized(clean)
|
||
split_half = split_half_jsd(d_norm)
|
||
if ref_cells:
|
||
dist_a = distributions_by_cell(d_norm)
|
||
entries, mean_jsd = compare_cells(dist_a, ref_cells)
|
||
outliers = [e for e in entries
|
||
if e['jsd'] > 0.5 and min(e['valid_a'], e['valid_b']) >= 15]
|
||
if mean_jsd is not None:
|
||
sh = split_half if split_half and split_half > 0 else 0.02
|
||
ratio = mean_jsd / max(sh, 0.02)
|
||
s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6)
|
||
if mean_jsd > 0.35:
|
||
s_val = min(s_val, 0.2)
|
||
s = {'s_dist': s_val, 'mean_jsd': mean_jsd,
|
||
'relative_ratio': round(ratio, 2), 'split_half': split_half,
|
||
'comparable_cells': len(entries), 'most_divergent': entries[:5],
|
||
'dist_outlier': bool(outliers),
|
||
'outlier_cells': [{'cell': o['cell'], 'jsd': round(o['jsd'], 3)}
|
||
for o in outliers]}
|
||
else:
|
||
s = {'s_dist': None, 'mean_jsd': None, 'comparable_cells': 0,
|
||
'dist_outlier': False, 'outlier_cells': []}
|
||
dist_cmp = {**s, 'baseline_p50': baseline_p50}
|
||
else:
|
||
dist_cmp = {'mean_jsd': None, 'split_half': split_half,
|
||
'baseline_p50': baseline_p50}
|
||
|
||
llmmap_tool = _load_llmmap_tool(args.tools_root) if args.llmmap_attribution else None
|
||
attribution = family_attribution(clean, aliases=aliases,
|
||
requested_family=req_family,
|
||
llmmap_tool=llmmap_tool)
|
||
|
||
# ---- adversarial 视图（注入态记录：ADV + 伪装条件下的 I/K 泄露扫描）----
|
||
adversarial = None
|
||
if injected:
|
||
adv_records = [r for r in injected if r.get('layer') == 'ADV']
|
||
adversarial = adversarial_signal(adv_records, all_records=injected,
|
||
requested_family=req_family,
|
||
dist_family=req_family, aliases=aliases,
|
||
mode='adversarial',
|
||
impersonate_role=args.impersonate)
|
||
|
||
report = build_report(clean, d_norm, dist_cmp, args.model, reference_info,
|
||
aliases, {'input': tokens_in, 'output': tokens_out},
|
||
elapsed, attribution=attribution, adversarial=adversarial,
|
||
mode='full')
|
||
# ---- variant 视图 ----
|
||
report['signals']['variant'] = variant_signal(
|
||
clean, logprobs_enabled=args.logprobs,
|
||
notes=['merged: graybox via Pass1 --logprobs' if args.logprobs
|
||
else 'merged: graybox off (vectron 不返回 logprobs; DS 后端 400 拒绝)',
|
||
'self-consistency via v_determinism_a/b'])
|
||
# ---- robustness 视图（温度轴 = clean 基线点 + sweep；语言/改写轴 = clean）----
|
||
report['signals']['robustness'] = robustness_signal(
|
||
clean + swept,
|
||
temperature_sweep=sorted({TEXT_TEMPERATURE} | set(sweep)))
|
||
report['passes'] = {name: {'probes': total, 'successful': ok}
|
||
for name, total, ok in passes}
|
||
report['logprobs_sampled'] = sum(1 for r in clean if r.get('top_logprobs'))
|
||
with open(report_path, 'w', encoding='utf-8') as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"[fp_fusion] mode=full verdict={report['verdict']} score={report['score']} | "
|
||
f"gate={report['gate']['quality']} "
|
||
f"({report['gate']['successful_probes']}/{report['gate']['total_probes']}) | "
|
||
f"passes=" + " ".join(f"{n}:{ok}/{t}" for n, t, ok in passes) +
|
||
f" | elapsed={elapsed:.0f}s")
|
||
fam = report['signals'].get('family') or {}
|
||
print(f"[fp_fusion] family: top1={fam.get('top1_family')} "
|
||
f"conf={fam.get('confidence')} s_fam={fam.get('s_fam')}")
|
||
if adversarial:
|
||
print(f"[fp_fusion] adv: impersonation={adversarial.get('impersonation_flag')} "
|
||
f"role_yield={adversarial.get('role_yield')} "
|
||
f"conflict={adversarial.get('claimed_behavior_conflict')}")
|
||
print(f"[fp_fusion] report: {report_path}\n[fp_fusion] raw: {raw_path}")
|
||
return report
|
||
|
||
|
||
def main(argv=None):
|
||
parser = argparse.ArgumentParser(
|
||
prog='fp_fusion',
|
||
description='FP-Fusion strict benchmark (v1.2, dims A/C)')
|
||
parser.add_argument('--api-url', required=True)
|
||
parser.add_argument('--model', required=True)
|
||
parser.add_argument('--report-path', required=True)
|
||
parser.add_argument('--timeout', type=int, default=120)
|
||
parser.add_argument('--tools-root', default=os.environ.get('FP_TOOLS_ROOT', '/data1/xii'))
|
||
parser.add_argument('--reference', default=None,
|
||
help='detector-schema reference JSON; omit = self mode')
|
||
parser.add_argument('--aliases', default=None, help='family_aliases.json override')
|
||
parser.add_argument('--d-samples', type=int, default=20)
|
||
parser.add_argument('--baseline-samples', type=int, default=20)
|
||
parser.add_argument('--text-limit', type=int, default=0, help='>0 只跑前 N 条文本探针(冒烟)')
|
||
parser.add_argument('--d-concurrency', type=int, default=4)
|
||
parser.add_argument('--text-concurrency', type=int, default=3)
|
||
parser.add_argument('--text-max-tokens', type=int, default=256)
|
||
# ---- 维度 A/C ----
|
||
parser.add_argument('--mode', choices=MODES, default='verify',
|
||
help='verify=原行为 | attribution=+家族归因 | adversarial=+对抗冒充 | '
|
||
'full=模式合并：三通道一次采集(clean/injected/sweep)，五视图离线打分 '
|
||
'(需 --impersonate 启用对抗视图)')
|
||
parser.add_argument('--impersonate', default=None,
|
||
help='对抗模式：注入伪装角色 system prompt（如 "You are GPT-4o..."）')
|
||
parser.add_argument('--logprobs', action='store_true', default=False,
|
||
help='灰盒预留：采集首个 token 的 top-logprobs（一期不评分）')
|
||
parser.add_argument('--llmmap-attribution', action='store_true', default=False,
|
||
help='启用 LLMmap 嵌入辅助归因（需 torch+e5）')
|
||
parser.add_argument('--extra-body', default=None,
|
||
help='附加请求体字段 JSON，如 {"thinking":{"type":"disabled"}}')
|
||
# ---- 维度 D ----
|
||
parser.add_argument('--temperature-sweep', default=None,
|
||
help='文本层温度扫描，逗号分隔如 "0.0,0.7,1.0"（robustness 用）')
|
||
parser.add_argument('--prompt-variants', type=int, default=0,
|
||
help='>0 时 D 层每 cell 轮换前 N 个 paraphrase（改写轴）')
|
||
# ---- 剪枝落地（2026-09-07 分析结论；默认关闭，保持旧行为）----
|
||
parser.add_argument('--cells', default='all',
|
||
help="'all'=全 26 cell(默认) | 'core16'=剪枝定稿集 | 逗号分隔 cell 清单")
|
||
parser.add_argument('--text-skip', default='none',
|
||
help="'none'=全 36 条(默认) | 'pruned7'=剪枝定稿 7 条 | 逗号分隔探针 id")
|
||
parser.add_argument('--api-key', default=None,
|
||
help='Bearer API key（vectron 等需要鉴权；本地端点可省略）')
|
||
args = parser.parse_args(argv)
|
||
|
||
report_path = Path(args.report_path).resolve()
|
||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||
raw_path = report_path.parent / 'raw_answers.jsonl'
|
||
|
||
extra_body = None
|
||
if args.extra_body:
|
||
try:
|
||
extra_body = json.loads(args.extra_body)
|
||
except json.JSONDecodeError as e:
|
||
print(f'ERROR: --extra-body 不是合法 JSON: {e}', file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
temp_sweep = None
|
||
if args.temperature_sweep:
|
||
temp_sweep = [float(x.strip()) for x in args.temperature_sweep.split(',')
|
||
if x.strip()]
|
||
|
||
# ---- 剪枝参数解析（--cells / --text-skip）----
|
||
if args.cells == 'all':
|
||
d_cells = None
|
||
elif args.cells == 'core16':
|
||
d_cells = set(CORE16_CELLS)
|
||
else:
|
||
d_cells = {x.strip() for x in args.cells.split(',') if x.strip()}
|
||
universe = {f"{c['id']}:{l}" for c in ALL_CELL_DEFS for l in ('en', 'zh')}
|
||
bad = d_cells - universe
|
||
if bad:
|
||
print(f'ERROR: --cells 含未知 cell: {sorted(bad)}', file=sys.stderr)
|
||
sys.exit(1)
|
||
if args.text_skip == 'none':
|
||
text_skip = set()
|
||
elif args.text_skip == 'pruned7':
|
||
text_skip = set(TEXT_PRUNED_V7)
|
||
else:
|
||
text_skip = {x.strip() for x in args.text_skip.split(',') if x.strip()}
|
||
known = {p['id'] for p in ALL_TEXT_PROBES}
|
||
bad = text_skip - known
|
||
if bad:
|
||
print(f'ERROR: --text-skip 含未知探针: {sorted(bad)}', file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
reference_info, ref_cells = None, None
|
||
if args.reference:
|
||
ref = load_reference(resolve_reference(args.reference))
|
||
reference_info, ref_cells = ref['model'], ref['cells']
|
||
|
||
# ---- 模式合并：三通道一次采集，五视图打分 ----
|
||
if args.mode == 'full':
|
||
_run_full(args, report_path, raw_path, extra_body, d_cells, text_skip,
|
||
reference_info, ref_cells)
|
||
return
|
||
|
||
# variant 模式必须开 logprobs（灰盒信号）
|
||
logprobs = args.logprobs or (args.mode == 'variant')
|
||
|
||
engine = FusionEngine(api_url=args.api_url, model=args.model, timeout=args.timeout,
|
||
d_samples=args.d_samples, baseline_samples=args.baseline_samples,
|
||
text_limit=args.text_limit, d_concurrency=args.d_concurrency,
|
||
text_concurrency=args.text_concurrency,
|
||
text_max_tokens=args.text_max_tokens,
|
||
extra_body=extra_body,
|
||
system_prompt_override=args.impersonate,
|
||
logprobs=logprobs,
|
||
temperature_sweep=temp_sweep,
|
||
prompt_variants=args.prompt_variants,
|
||
d_cells=d_cells,
|
||
api_key=args.api_key)
|
||
|
||
probes = _assemble_probes(args.mode, args.impersonate, text_skip)
|
||
n_cells = 26 if d_cells is None else len(d_cells)
|
||
print(f"[fp_fusion] battery: cells={n_cells}/26 (D={n_cells * args.d_samples} req) "
|
||
f"text probes={len(probes)} baseline={args.baseline_samples}")
|
||
|
||
t0 = time.monotonic()
|
||
records = asyncio.run(engine.run(probes))
|
||
elapsed = time.monotonic() - t0
|
||
|
||
with open(raw_path, 'w', encoding='utf-8') as f:
|
||
for r in records:
|
||
f.write(json.dumps(r, ensure_ascii=False) + '\n')
|
||
|
||
d_norm = build_d_normalized(records)
|
||
split_half = split_half_jsd(d_norm)
|
||
|
||
if ref_cells:
|
||
dist_a = distributions_by_cell(d_norm)
|
||
entries, mean_jsd = compare_cells(dist_a, ref_cells)
|
||
# v1.1 dist_outlier 规则: 单 cell 极端分化(双方≥15有效且JSD>0.5)
|
||
outliers = [e for e in entries
|
||
if e['jsd'] > 0.5 and min(e['valid_a'], e['valid_b']) >= 15]
|
||
s = dict()
|
||
if mean_jsd is not None:
|
||
sh = split_half if split_half and split_half > 0 else 0.02
|
||
ratio = mean_jsd / max(sh, 0.02)
|
||
s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6)
|
||
if mean_jsd > 0.35:
|
||
s_val = min(s_val, 0.2)
|
||
s = {'s_dist': s_val, 'mean_jsd': mean_jsd,
|
||
'relative_ratio': round(ratio, 2),
|
||
'split_half': split_half,
|
||
'comparable_cells': len(entries),
|
||
'most_divergent': entries[:5],
|
||
'dist_outlier': bool(outliers),
|
||
'outlier_cells': [{'cell': o['cell'], 'jsd': round(o['jsd'], 3)}
|
||
for o in outliers]}
|
||
else:
|
||
s = {'s_dist': None, 'mean_jsd': None, 'comparable_cells': 0,
|
||
'dist_outlier': False, 'outlier_cells': [],
|
||
'note': 'no comparable cells (valid samples too few)'}
|
||
dist_cmp = {**s, 'baseline_p50': engine.baseline_p50}
|
||
else:
|
||
dist_cmp = {'mean_jsd': None, 'split_half': split_half,
|
||
'baseline_p50': engine.baseline_p50}
|
||
|
||
aliases = load_aliases(args.aliases)
|
||
req_family = None
|
||
if args.mode != 'verify':
|
||
from .scorer import requested_family
|
||
req_family = requested_family(args.model, aliases)
|
||
|
||
# ---- 维度 A：家族归因 ----
|
||
attribution = None
|
||
if args.mode != 'verify':
|
||
from .attribution import family_attribution
|
||
llmmap_tool = _load_llmmap_tool(args.tools_root) if args.llmmap_attribution else None
|
||
attribution = family_attribution(records, aliases=aliases,
|
||
requested_family=req_family,
|
||
llmmap_tool=llmmap_tool)
|
||
|
||
# ---- 维度 C：对抗信号 ----
|
||
adversarial = None
|
||
if args.mode == 'adversarial':
|
||
from .probes_adv import adversarial_signal
|
||
adv_records = [r for r in records if r.get('layer') == 'ADV']
|
||
dist_family = req_family
|
||
adversarial = adversarial_signal(adv_records, all_records=records,
|
||
requested_family=req_family,
|
||
dist_family=dist_family,
|
||
aliases=aliases, mode=args.mode,
|
||
impersonate_role=args.impersonate)
|
||
|
||
report = build_report(records, d_norm, dist_cmp, args.model, reference_info,
|
||
aliases, {'input': engine.tokens_in, 'output': engine.tokens_out},
|
||
elapsed, attribution=attribution, adversarial=adversarial,
|
||
mode=args.mode)
|
||
|
||
# ---- 维度 B：变体区分信号 ----
|
||
if args.mode == 'variant':
|
||
from .probes_variant import variant_signal
|
||
report['signals']['variant'] = variant_signal(
|
||
records, logprobs_enabled=logprobs,
|
||
notes=['graybox via --logprobs', 'self-consistency via v_determinism_a/b'])
|
||
|
||
# ---- 维度 D：鲁棒性信号 ----
|
||
if args.mode == 'robustness':
|
||
from .scorer import robustness_signal
|
||
report['signals']['robustness'] = robustness_signal(
|
||
records, temperature_sweep=temp_sweep)
|
||
|
||
if logprobs:
|
||
report['logprobs_sampled'] = sum(1 for r in records if r.get('top_logprobs'))
|
||
with open(report_path, 'w', encoding='utf-8') as f:
|
||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||
|
||
print(f"[fp_fusion] mode={report['mode']} verdict={report['verdict']} "
|
||
f"score={report['score']} | gate={report['gate']['quality']} "
|
||
f"({report['gate']['successful_probes']}/{report['gate']['total_probes']}) | "
|
||
f"meanJSD={report['signals']['dist'].get('mean_jsd')} | "
|
||
f"latency p50={engine.baseline_p50}ms elapsed={elapsed:.0f}s")
|
||
if attribution:
|
||
print(f"[fp_fusion] family: top1={attribution.get('top1_family')} "
|
||
f"conf={attribution.get('confidence')} s_fam={attribution.get('s_fam')}")
|
||
if adversarial:
|
||
print(f"[fp_fusion] adv: impersonation={adversarial.get('impersonation_flag')} "
|
||
f"role_yield={adversarial.get('role_yield')} "
|
||
f"conflict={adversarial.get('claimed_behavior_conflict')}")
|
||
print(f"[fp_fusion] report: {report_path}\n[fp_fusion] raw: {raw_path}")
|
||
return 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main(sys.argv[1:])) |