370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
90 lines
4.8 KiB
Python
90 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
||
"""剪枝落地离线验证（零 API）：
|
||
1. 组装计数：core16/pruned7 → D=400、文本=29；默认参数 → 650/36（向后兼容）
|
||
2. 9 模型 slim 电池全管线重放 vs 存档全量 verify.json → 判决/分数变化
|
||
"""
|
||
import json
|
||
import sys
|
||
|
||
from pathlib import Path # noqa: E402
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # 仓库根/site-packages, 使 evalharness 包可导入
|
||
from evalharness.fingerprint.battery import ALL_TEXT_PROBES, CORE16_CELLS, TEXT_PRUNED_V7
|
||
from evalharness.fingerprint.engine import (FusionEngine, build_d_normalized, compare_cells, # noqa: E402
|
||
distributions_by_cell, load_reference, split_half_jsd)
|
||
from evalharness.fingerprint.run_fp_fusion import _assemble_probes
|
||
from evalharness.fingerprint.scorer import build_report, load_aliases
|
||
|
||
BFD = "/tmp/bfd"
|
||
R = str(Path(__file__).resolve().parent / 'references')
|
||
MODELS = [
|
||
("deepseek_v4_flash", "deepseek_v4_flash_fusion_reference.json", "DeepSeek/DeepSeek-V4-Flash"),
|
||
("deepseek_v4_flash_0731", "deepseek_v4_flash_0731_fusion_reference.json", "DeepSeek/DeepSeek-V4-Flash-0731"),
|
||
("deepseek_v4_pro", "deepseek_v4_pro_fusion_reference.json", "DeepSeek/DeepSeek-V4-Pro"),
|
||
("glm_51", "glm_51_fusion_reference.json", "ZhipuAi/GLM-5.1"),
|
||
("glm_52", "glm52_vectron_fusion_reference.json", "ZhipuAi/GLM-5.2"),
|
||
("glm_53", "glm53_fusion_reference.json", "ZhipuAi/GLM-5.3"),
|
||
("kimi_k2_6", "kimi_k2_6_fusion_reference.json", "MoonshotAi/Kimi-K2.6"),
|
||
("kimi_k2_7code", "kimi_k2_7code_fusion_reference.json", "MoonshotAi/Kimi-K2.7-Code"),
|
||
("kimi_k3", "kimi_k3_fusion_reference.json", "MoonshotAi/Kimi-K3"),
|
||
]
|
||
|
||
# ---------- 1. 组装计数 ----------
|
||
p_slim = _assemble_probes("verify", None, set(TEXT_PRUNED_V7))
|
||
p_full = _assemble_probes("verify", None, None)
|
||
eng_slim = FusionEngine(api_url="http://x", model="m", d_samples=25,
|
||
d_cells=set(CORE16_CELLS))
|
||
eng_full = FusionEngine(api_url="http://x", model="m", d_samples=25)
|
||
j_slim, j_full = eng_slim._d_jobs(), eng_full._d_jobs()
|
||
print(f"组装验证: slim 文本 {len(p_slim)}(期望29) D {len(j_slim)}(期望400) | "
|
||
f"默认 文本 {len(p_full)}(期望36) D {len(j_full)}(期望650)")
|
||
cells_seen = {j[0] for j in j_slim}
|
||
assert cells_seen == set(CORE16_CELLS), f"cell 集合不符: {cells_seen ^ set(CORE16_CELLS)}"
|
||
ids_seen = {p["id"] for p in p_slim}
|
||
assert not (ids_seen & set(TEXT_PRUNED_V7)), "剪除探针泄漏"
|
||
print("断言通过: cell 集合=core16, 无剪除探针泄漏\n")
|
||
|
||
# ---------- 2. 9 模型 slim 重放 ----------
|
||
C16, P7 = set(CORE16_CELLS), set(TEXT_PRUNED_V7)
|
||
aliases = load_aliases(None)
|
||
print(f"{'模型':24s} {'全量判决/分':>22s} {'slim判决/分':>22s} {'Δscore':>8s} 判决")
|
||
flips = 0
|
||
for d, rf, mid in MODELS:
|
||
recs = [json.loads(l) for l in open(f"{BFD}/{d}/raw_answers.jsonl")]
|
||
verify = json.load(open(f"{BFD}/{d}/verify.json"))
|
||
ref = load_reference(f"{R}/{rf}")
|
||
slim = [r for r in recs if
|
||
(r.get("layer") == "D" and r["cell"] in C16) or
|
||
(r.get("layer") in ("I", "K", "C", "S") and r["id"] not in P7) or
|
||
(r.get("layer") not in ("D", "I", "K", "C", "S"))]
|
||
dn = build_d_normalized(slim)
|
||
sh = split_half_jsd(dn)
|
||
dist = distributions_by_cell(dn)
|
||
entries, mean_jsd = compare_cells(dist, ref["cells"])
|
||
outliers = [e for e in entries
|
||
if e["jsd"] > 0.5 and min(e["valid_a"], e["valid_b"]) >= 15]
|
||
if mean_jsd is not None:
|
||
ratio = mean_jsd / max(sh or 0.02, 0.02)
|
||
s_val = 1.0 if ratio < 2 else (0.0 if ratio > 8 else 1.0 - (ratio - 2) / 6)
|
||
if mean_jsd > 0.35:
|
||
s_val = min(s_val, 0.2)
|
||
s = {"s_dist": s_val, "mean_jsd": mean_jsd, "relative_ratio": round(ratio, 2),
|
||
"split_half": sh, "comparable_cells": len(entries),
|
||
"most_divergent": entries[:5], "dist_outlier": bool(outliers),
|
||
"outlier_cells": [{"cell": o["cell"], "jsd": round(o["jsd"], 3)}
|
||
for o in outliers]}
|
||
else:
|
||
s = {"s_dist": None, "mean_jsd": None, "comparable_cells": 0,
|
||
"dist_outlier": False, "outlier_cells": []}
|
||
dist_cmp = {**s, "baseline_p50": verify["signals"]["dist"].get("baseline_p50")}
|
||
rpt = build_report(slim, dn, dist_cmp, mid, ref["model"], aliases,
|
||
verify.get("tokens_used") or {},
|
||
verify.get("elapsed_s") or 0.0,
|
||
attribution=None, adversarial=None, mode="verify")
|
||
dv = rpt["verdict"] == verify["verdict"]
|
||
flips += not dv
|
||
print(f"{d:24s} {verify['verdict'] + ' ' + format(verify['score'], '.4f'):>22s} "
|
||
f"{rpt['verdict'] + ' ' + format(rpt['score'], '.4f'):>22s} "
|
||
f"{rpt['score'] - verify['score']:+8.4f} {'✓' if dv else '✗ 翻转'}")
|
||
print(f"\n判决翻转: {flips}/9")
|