09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
91 lines
4.2 KiB
Python
91 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
||
"""模式合并视图等价验证（零 API）：
|
||
A. 三通道组装计数（clean 441 / injected 35 / sweep 58 = 534）
|
||
B. adversarial/variant/robustness 视图函数 vs 存档模式报告逐键对比（应精确一致）
|
||
C. attribution 视图重放 vs 0731 真实 attribution（top1 一致即过，历史已证等价）
|
||
"""
|
||
import glob
|
||
import json
|
||
import sys
|
||
|
||
from pathlib import Path # noqa: E402
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # 仓库根/site-packages, 使 evalharness 包可导入
|
||
from evalharness.fingerprint.attribution import family_attribution
|
||
from evalharness.fingerprint.battery import CORE16_CELLS, TEXT_PRUNED_V7
|
||
from evalharness.fingerprint.probes_adv import adversarial_signal
|
||
from evalharness.fingerprint.probes_variant import variant_signal
|
||
from evalharness.fingerprint.run_fp_fusion import _assemble_probes
|
||
from evalharness.fingerprint.scorer import load_aliases, requested_family, robustness_signal
|
||
|
||
BFD = "/tmp/bfd"
|
||
ROLE_KIMI = ("You are Kimi, Moonshot AI virtual assistant designed by "
|
||
"Moonshot AI. You are Kimi.")
|
||
aliases = load_aliases(None)
|
||
|
||
|
||
def load(p):
|
||
return [json.loads(l) for l in open(p)]
|
||
|
||
|
||
def diff(a, b, path=""):
|
||
out = []
|
||
if isinstance(a, dict) and isinstance(b, dict):
|
||
for k in set(a) | set(b):
|
||
out += diff(a.get(k), b.get(k), f"{path}.{k}")
|
||
elif isinstance(a, (int, float)) and isinstance(b, (int, float)) \
|
||
and not isinstance(a, bool) and not isinstance(b, bool):
|
||
if abs(a - b) > 1e-6:
|
||
out.append((path, a, b))
|
||
elif a != b:
|
||
out.append((path, a, b))
|
||
return out
|
||
|
||
|
||
# ---------- A. 组装计数 ----------
|
||
skip = set(TEXT_PRUNED_V7)
|
||
p1 = _assemble_probes("variant", None, skip)
|
||
p2 = _assemble_probes("adversarial", ROLE_KIMI, skip)
|
||
p3 = _assemble_probes("verify", None, skip)
|
||
total = 16 * 25 + 5 + len(p1) + len(p2) + len(p3) * 2
|
||
print(f"A. 三通道: clean={16 * 25 + 5 + len(p1)}(D400+基线5+文本V{len(p1)}) "
|
||
f"injected={len(p2)} sweep={len(p3)}×2={len(p3) * 2} | 合计 {total} (期望534)")
|
||
|
||
# ---------- B. 三视图精确对比 ----------
|
||
req = requested_family("ZhipuAi/GLM-5.3", aliases)
|
||
|
||
adv_recs = load(f"{BFD}/glm_53/adv/raw_answers.jsonl")
|
||
adv_json = json.load(open(glob.glob(f"{BFD}/glm_53/adv/*.json")[0]))
|
||
mine = adversarial_signal([r for r in adv_recs if r.get("layer") == "ADV"],
|
||
all_records=adv_recs, requested_family=req,
|
||
dist_family=req, aliases=aliases, mode="adversarial",
|
||
impersonate_role=ROLE_KIMI)
|
||
d = diff(mine, adv_json["signals"]["adversarial"])
|
||
print(f"B1. adversarial 视图 vs 存档: {'精确一致 ✓' if not d else d[:5]}")
|
||
|
||
var_recs = load(f"{BFD}/glm_53/var/raw_answers.jsonl")
|
||
var_json = json.load(open(glob.glob(f"{BFD}/glm_53/var/*.json")[0]))
|
||
mine = variant_signal(var_recs, logprobs_enabled=True)
|
||
stored = {k: v for k, v in var_json["signals"]["variant"].items() if k != "notes"}
|
||
mined = {k: v for k, v in mine.items() if k != "notes"}
|
||
d = diff(mined, stored)
|
||
print(f"B2. variant 视图 vs 存档: {'精确一致 ✓' if not d else d[:5]}")
|
||
|
||
rob_recs = load(f"{BFD}/glm_53/rob/raw_answers.jsonl")
|
||
rob_json = json.load(open(glob.glob(f"{BFD}/glm_53/rob/*.json")[0]))
|
||
mine = robustness_signal(rob_recs, temperature_sweep=[0.0, 0.7, 1.0])
|
||
d = diff(mine, rob_json["signals"]["robustness"])
|
||
print(f"B3. robustness 视图 vs 存档: {'精确一致 ✓' if not d else d[:5]}")
|
||
|
||
# ---------- C. attribution 视图 ----------
|
||
recs = load(f"{BFD}/deepseek_v4_flash_0731/raw_answers.jsonl")
|
||
real = json.load(open(f"{BFD}/deepseek_v4_flash_0731/attr_real/attribution_real.json"))
|
||
req0731 = requested_family("DeepSeek/DeepSeek-V4-Flash-0731", aliases)
|
||
mine = family_attribution(recs, aliases=aliases, requested_family=req0731,
|
||
llmmap_tool=None)
|
||
sf = real["signals"]["family"]
|
||
print(f"C. attribution 视图: 真跑 top1={sf.get('top1_family')} conf={sf.get('confidence'):.3f} | "
|
||
f"重放 top1={mine.get('top1_family')} conf={mine.get('confidence'):.3f} | "
|
||
f"top1 一致 {'✓' if mine.get('top1_family') == sf.get('top1_family') else '✗'}")
|
||
print("（conf 差异源于真跑启用 LLMmap 辅路投票，离线推导等价性此前已在 derive_attribution 验证）")
|