#!/usr/bin/env python3 """剪枝落地离线验证(零 API): 1. 组装计数:core16/pruned7 → D=400、文本=29;默认参数 → 650/36(向后兼容) 2. 9 模型 slim 电池全管线重放 vs 存档全量 verify.json → 判决/分数变化 """ import json import sys from pathlib import Path # noqa: E402 sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # 仓库根/site-packages, 使 evalharness 包可导入 from evalharness.fingerprint.battery import ALL_TEXT_PROBES, CORE16_CELLS, TEXT_PRUNED_V7 from evalharness.fingerprint.engine import (FusionEngine, build_d_normalized, compare_cells, # noqa: E402 distributions_by_cell, load_reference, split_half_jsd) from evalharness.fingerprint.run_fp_fusion import _assemble_probes from evalharness.fingerprint.scorer import build_report, load_aliases BFD = "/tmp/bfd" R = str(Path(__file__).resolve().parent / 'references') MODELS = [ ("deepseek_v4_flash", "deepseek_v4_flash_fusion_reference.json", "DeepSeek/DeepSeek-V4-Flash"), ("deepseek_v4_flash_0731", "deepseek_v4_flash_0731_fusion_reference.json", "DeepSeek/DeepSeek-V4-Flash-0731"), ("deepseek_v4_pro", "deepseek_v4_pro_fusion_reference.json", "DeepSeek/DeepSeek-V4-Pro"), ("glm_51", "glm_51_fusion_reference.json", "ZhipuAi/GLM-5.1"), ("glm_52", "glm52_vectron_fusion_reference.json", "ZhipuAi/GLM-5.2"), ("glm_53", "glm53_fusion_reference.json", "ZhipuAi/GLM-5.3"), ("kimi_k2_6", "kimi_k2_6_fusion_reference.json", "MoonshotAi/Kimi-K2.6"), ("kimi_k2_7code", "kimi_k2_7code_fusion_reference.json", "MoonshotAi/Kimi-K2.7-Code"), ("kimi_k3", "kimi_k3_fusion_reference.json", "MoonshotAi/Kimi-K3"), ] # ---------- 1. 组装计数 ---------- p_slim = _assemble_probes("verify", None, set(TEXT_PRUNED_V7)) p_full = _assemble_probes("verify", None, None) eng_slim = FusionEngine(api_url="http://x", model="m", d_samples=25, d_cells=set(CORE16_CELLS)) eng_full = FusionEngine(api_url="http://x", model="m", d_samples=25) j_slim, j_full = eng_slim._d_jobs(), eng_full._d_jobs() print(f"组装验证: slim 文本 {len(p_slim)}(期望29) D {len(j_slim)}(期望400) | " f"默认 文本 {len(p_full)}(期望36) D {len(j_full)}(期望650)") cells_seen = {j[0] for j in j_slim} assert cells_seen == set(CORE16_CELLS), f"cell 集合不符: {cells_seen ^ set(CORE16_CELLS)}" ids_seen = {p["id"] for p in p_slim} assert not (ids_seen & set(TEXT_PRUNED_V7)), "剪除探针泄漏" print("断言通过: cell 集合=core16, 无剪除探针泄漏\n") # ---------- 2. 9 模型 slim 重放 ---------- C16, P7 = set(CORE16_CELLS), set(TEXT_PRUNED_V7) aliases = load_aliases(None) print(f"{'模型':24s} {'全量判决/分':>22s} {'slim判决/分':>22s} {'Δscore':>8s} 判决") flips = 0 for d, rf, mid in MODELS: recs = [json.loads(l) for l in open(f"{BFD}/{d}/raw_answers.jsonl")] verify = json.load(open(f"{BFD}/{d}/verify.json")) ref = load_reference(f"{R}/{rf}") slim = [r for r in recs if (r.get("layer") == "D" and r["cell"] in C16) or (r.get("layer") in ("I", "K", "C", "S") and r["id"] not in P7) or (r.get("layer") not in ("D", "I", "K", "C", "S"))] dn = build_d_normalized(slim) sh = split_half_jsd(dn) dist = distributions_by_cell(dn) entries, mean_jsd = compare_cells(dist, ref["cells"]) outliers = [e for e in entries if e["jsd"] > 0.5 and min(e["valid_a"], e["valid_b"]) >= 15] if mean_jsd is not None: ratio = mean_jsd / max(sh or 0.02, 0.02) s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6) if mean_jsd > 0.35: s_val = min(s_val, 0.2) s = {"s_dist": s_val, "mean_jsd": mean_jsd, "relative_ratio": round(ratio, 2), "split_half": sh, "comparable_cells": len(entries), "most_divergent": entries[:5], "dist_outlier": bool(outliers), "outlier_cells": [{"cell": o["cell"], "jsd": round(o["jsd"], 3)} for o in outliers]} else: s = {"s_dist": None, "mean_jsd": None, "comparable_cells": 0, "dist_outlier": False, "outlier_cells": []} dist_cmp = {**s, "baseline_p50": verify["signals"]["dist"].get("baseline_p50")} rpt = build_report(slim, dn, dist_cmp, mid, ref["model"], aliases, verify.get("tokens_used") or {}, verify.get("elapsed_s") or 0.0, attribution=None, adversarial=None, mode="verify") dv = rpt["verdict"] == verify["verdict"] flips += not dv print(f"{d:24s} {verify['verdict'] + ' ' + format(verify['score'], '.4f'):>22s} " f"{rpt['verdict'] + ' ' + format(rpt['score'], '.4f'):>22s} " f"{rpt['score'] - verify['score']:+8.4f} {'✓' if dv else '✗ 翻转'}") print(f"\n判决翻转: {flips}/9")