370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
165 lines
7.7 KiB
Python
165 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
||
"""汇总四维×9 能力矩阵 CSV + 一致性校验（反向验证 / attribution 等价性）。"""
|
||
import csv
|
||
import json
|
||
import os
|
||
|
||
BFD = "/tmp/bfd"
|
||
|
||
# (短名, 模型ID, 参考文件, 参考口径)
|
||
MODELS = [
|
||
("deepseek_v4_flash", "DeepSeek/DeepSeek-V4-Flash", "deepseek_v4_flash_fusion_reference.json", "旧key"),
|
||
("deepseek_v4_flash_0731", "DeepSeek/DeepSeek-V4-Flash-0731", "deepseek_v4_flash_0731_fusion_reference.json", "旧key"),
|
||
("deepseek_v4_pro", "DeepSeek/DeepSeek-V4-Pro", "deepseek_v4_pro_fusion_reference.json", "旧key"),
|
||
("glm_51", "ZhipuAi/GLM-5.1", "glm_51_fusion_reference.json", "新key"),
|
||
("glm_52", "ZhipuAi/GLM-5.2", "glm52_vectron_fusion_reference.json", "旧key"),
|
||
("glm_53", "ZhipuAi/GLM-5.3", "glm53_fusion_reference.json", "旧key"),
|
||
("kimi_k2_6", "MoonshotAi/Kimi-K2.6", "kimi_k2_6_fusion_reference.json", "新key"),
|
||
("kimi_k2_7code", "MoonshotAi/Kimi-K2.7-Code", "kimi_k2_7code_fusion_reference.json", "新key"),
|
||
("kimi_k3", "MoonshotAi/Kimi-K3", "kimi_k3_fusion_reference.json", "旧key"),
|
||
]
|
||
REPRESENTATIVES = ("glm_53", "kimi_k3", "deepseek_v4_pro")
|
||
|
||
|
||
def load(path):
|
||
if os.path.exists(path):
|
||
try:
|
||
return json.load(open(path))
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
|
||
def fmt(x, nd=3):
|
||
if x is None:
|
||
return ""
|
||
if isinstance(x, float):
|
||
return f"{x:.{nd}f}"
|
||
return str(x)
|
||
|
||
|
||
rows = []
|
||
for name, mid, ref,口径 in MODELS:
|
||
d = os.path.join(BFD, name)
|
||
v = load(f"{d}/verify.json")
|
||
a = load(f"{d}/attribution.json")
|
||
adv = load(f"{d}/adv/adversarial.json")
|
||
var = load(f"{d}/var/variant.json")
|
||
rob = load(f"{d}/rob/robustness.json")
|
||
|
||
# 识别（verify）
|
||
if v:
|
||
sig = v.get("signals", {})
|
||
ident = {"verdict": v.get("verdict"), "score": v.get("score"),
|
||
"meanJSD": sig.get("dist", {}).get("mean_jsd"),
|
||
"gate": v.get("gate", {}).get("quality"),
|
||
"s_idn": sig.get("identity", {}).get("s_idn")}
|
||
else:
|
||
ident = dict.fromkeys(("verdict", "score", "meanJSD", "gate", "s_idn"))
|
||
|
||
# 归因（attribution）
|
||
if a:
|
||
fam = a.get("signals", {}).get("family") or {}
|
||
attr = {"verdict": a.get("verdict"), "score": a.get("score"),
|
||
"top1": fam.get("top1_family"), "conf": fam.get("confidence"),
|
||
"s_fam": fam.get("s_fam"), "conflict": fam.get("conflict"),
|
||
"req": mid.split("/")[0].replace("ZhipuAi", "glm")
|
||
.replace("MoonshotAi", "kimi").replace("DeepSeek", "deepseek")}
|
||
else:
|
||
attr = dict.fromkeys(("verdict", "score", "top1", "conf", "s_fam", "conflict", "req"))
|
||
|
||
# 对抗（adversarial，仅 3 代表）
|
||
if adv:
|
||
sig = adv.get("signals", {})
|
||
av = sig.get("adversarial") or {}
|
||
ad = {"imp_flag": av.get("impersonation_flag"),
|
||
"role_yield": av.get("role_yield"),
|
||
"conflict": av.get("claimed_behavior_conflict"),
|
||
"style_suspect": av.get("style_imitation_suspect")}
|
||
else:
|
||
ad = dict.fromkeys(("imp_flag", "role_yield", "conflict", "style_suspect"))
|
||
ad["imp_flag"] = "未跑(3代表抽样)" if name not in REPRESENTATIVES else "待跑"
|
||
|
||
# 变体（variant，仅 3 代表）
|
||
if var:
|
||
vs = (var.get("signals", {}).get("variant") or {})
|
||
vt = {"graybox": vs.get("graybox_present"),
|
||
"top1_stab": vs.get("top1_stability"),
|
||
"self_jsd": vs.get("self_consistency_jsd"),
|
||
"logprob_mean": vs.get("logprob_mean")}
|
||
else:
|
||
vt = dict.fromkeys(("graybox", "top1_stab", "self_jsd", "logprob_mean"))
|
||
vt["graybox"] = "未跑(3代表抽样)" if name not in REPRESENTATIVES else "待跑"
|
||
|
||
# 鲁棒三轴（robustness，仅 3 代表）
|
||
if rob:
|
||
rs = (rob.get("signals", {}).get("robustness") or {})
|
||
ta = rs.get("temp_axis") if isinstance(rs.get("temp_axis"), dict) else {}
|
||
la = rs.get("lang_axis") if isinstance(rs.get("lang_axis"), dict) else {}
|
||
pa = rs.get("paraphrase_axis") if isinstance(rs.get("paraphrase_axis"), dict) else {}
|
||
rb = {"temp": fmt(ta.get("mean_consistency")),
|
||
"lang": fmt(la.get("mean_consistency")),
|
||
"para": fmt(pa.get("mean_jsd"))}
|
||
else:
|
||
rb = {"temp": "未跑" if name not in REPRESENTATIVES else "待跑",
|
||
"lang": "未跑" if name not in REPRESENTATIVES else "待跑",
|
||
"para": "未跑" if name not in REPRESENTATIVES else "待跑"}
|
||
|
||
rows.append({
|
||
"模型": mid, "短名": name, "参考口径": 口径,
|
||
"识别_verdict": ident["verdict"], "识别_score": fmt(ident["score"]),
|
||
"识别_meanJSD": fmt(ident["meanJSD"]), "识别_gate": ident["gate"],
|
||
"归因_verdict": attr["verdict"], "归因_score": fmt(attr["score"]),
|
||
"归因_top1": attr["top1"], "归因_conf": fmt(attr["conf"]),
|
||
"归因_s_fam": fmt(attr["s_fam"]), "归因_冲突": attr["conflict"],
|
||
"对抗_冒充实锤": ad["imp_flag"], "对抗_角色屈服": fmt(ad["role_yield"]),
|
||
"对抗_声称行为矛盾": fmt(ad["conflict"]), "对抗_风格模仿嫌疑": fmt(ad["style_suspect"]),
|
||
"变体_灰盒": vt["graybox"], "变体_top1稳定": fmt(vt["top1_stab"]),
|
||
"变体_自一致JSD": fmt(vt["self_jsd"]), "变体_logprob均值": fmt(vt["logprob_mean"]),
|
||
"鲁棒_温度轴": rb["temp"], "鲁棒_语言轴": rb["lang"], "鲁棒_改写轴JSD": rb["para"],
|
||
})
|
||
|
||
out_csv = os.path.join(BFD, "能力矩阵.csv")
|
||
with open(out_csv, "w", newline="", encoding="utf-8-sig") as f:
|
||
w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||
w.writeheader()
|
||
w.writerows(rows)
|
||
print(f"written {out_csv} rows={len(rows)}")
|
||
|
||
# ---- 校验 1：反向验证（glm_53 verify 复跑 score 差 < 0.05）----
|
||
v1 = load(f"{BFD}/glm_53/verify.json")
|
||
v2 = load(f"{BFD}/glm_53/rerun/verify_rerun.json")
|
||
if v1 and v2:
|
||
diff = abs(v1["score"] - v2["score"])
|
||
print(f"[反向验证] glm_53 首跑={v1['score']} 复跑={v2['score']} 差={diff:.4f} "
|
||
f"{'PASS(<0.05)' if diff < 0.05 else 'FAIL(>=0.05)'}")
|
||
else:
|
||
print("[反向验证] glm_53 复跑尚未完成")
|
||
|
||
# ---- 校验 2：attribution 等价性（0731 真实运行 vs 离线推导）----
|
||
real = load(f"{BFD}/deepseek_v4_flash_0731/attr_real/attribution_real.json")
|
||
derived = load(f"{BFD}/deepseek_v4_flash_0731/attribution.json")
|
||
if real and derived:
|
||
rf = (real.get("signals", {}).get("family") or {})
|
||
df_ = (derived.get("signals", {}).get("family") or {})
|
||
same_top1 = rf.get("top1_family") == df_.get("top1_family")
|
||
print(f"[attribution 等价性] 0731 真实: top1={rf.get('top1_family')} conf={rf.get('confidence')} "
|
||
f"score={real.get('score')} | 离线: top1={df_.get('top1_family')} conf={df_.get('confidence')} "
|
||
f"score={derived.get('score')} → top1一致={same_top1}")
|
||
else:
|
||
print("[attribution 等价性] 0731 真实运行尚未完成")
|
||
|
||
# ---- 校验 3：variant 灰盒（3 代表 graybox_present 均 True 且 logprob_mean 有值）----
|
||
ok = 0
|
||
for name in REPRESENTATIVES:
|
||
var = load(f"{BFD}/{name}/var/variant.json")
|
||
if var:
|
||
vs = var.get("signals", {}).get("variant") or {}
|
||
print(f"[variant 灰盒] {name}: graybox={vs.get('graybox_present')} "
|
||
f"logprob_mean={vs.get('logprob_mean')}")
|
||
if vs.get("graybox_present") and vs.get("logprob_mean") is not None:
|
||
ok += 1
|
||
else:
|
||
print(f"[variant 灰盒] {name}: 待跑")
|
||
print(f"[variant 灰盒] 通过 {ok}/3")
|