370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
101 lines
3.9 KiB
Python
101 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
||
"""9×9 交叉混淆矩阵：每个模型实测 D 层分布 vs 全部 9 个参考（纯离线）。
|
||
|
||
混淆判定：某模型的全部 JSD 里，最低者（top-1）若不是自己的参考 → 记一次混淆。
|
||
附带：glm_53 复跑样本作为第二独立样本验证比对稳定性。
|
||
"""
|
||
import json
|
||
import sys
|
||
|
||
from pathlib import Path # noqa: E402
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # 仓库根/site-packages, 使 evalharness 包可导入
|
||
from evalharness.fingerprint.engine import (build_d_normalized, compare_cells, # noqa: E402
|
||
distributions_by_cell, load_reference)
|
||
|
||
BFD = "/tmp/bfd"
|
||
R = str(Path(__file__).resolve().parent / 'references')
|
||
|
||
MODELS = [
|
||
("deepseek_v4_flash", "deepseek_v4_flash_fusion_reference.json", "DS"),
|
||
("deepseek_v4_flash_0731", "deepseek_v4_flash_0731_fusion_reference.json", "DS"),
|
||
("deepseek_v4_pro", "deepseek_v4_pro_fusion_reference.json", "DS"),
|
||
("glm_51", "glm_51_fusion_reference.json", "GLM"),
|
||
("glm_52", "glm52_vectron_fusion_reference.json", "GLM"),
|
||
("glm_53", "glm53_fusion_reference.json", "GLM"),
|
||
("kimi_k2_6", "kimi_k2_6_fusion_reference.json", "Kimi"),
|
||
("kimi_k2_7code", "kimi_k2_7code_fusion_reference.json", "Kimi"),
|
||
("kimi_k3", "kimi_k3_fusion_reference.json", "Kimi"),
|
||
]
|
||
FAMILY = {n: f for n, _, f in MODELS}
|
||
|
||
refs = {}
|
||
for n, rf, _ in MODELS:
|
||
refs[n] = load_reference(f"{R}/{rf}")["cells"]
|
||
|
||
|
||
def dist_of(path):
|
||
records = [json.loads(l) for l in open(path)]
|
||
d = build_d_normalized(records)
|
||
return distributions_by_cell(d)
|
||
|
||
|
||
live = {n: dist_of(f"{BFD}/{n}/raw_answers.jsonl") for n, _, _ in MODELS}
|
||
extra = {}
|
||
try:
|
||
extra["glm_53#rerun"] = dist_of(f"{BFD}/glm_53/rerun/raw_answers.jsonl")
|
||
except FileNotFoundError:
|
||
pass
|
||
|
||
names = [n for n, _, _ in MODELS]
|
||
results = {}
|
||
for lname, dist in {**live, **extra}.items():
|
||
results[lname] = {}
|
||
for rname, ref_cells in refs.items():
|
||
entries, mean_jsd = compare_cells(dist, ref_cells)
|
||
results[lname][rname] = (mean_jsd, len(entries))
|
||
|
||
print("JSD 矩阵（行=实测模型，列=参考，越低越像）:")
|
||
print("live\\ref".ljust(22) + "".join(n[:13].rjust(14) for n in names))
|
||
for lname in results:
|
||
print(lname[:21].ljust(22) +
|
||
"".join(f"{results[lname][n][0]:.3f}".rjust(14)
|
||
if results[lname][n][0] is not None else "—".rjust(14)
|
||
for n in names))
|
||
|
||
print()
|
||
print("=" * 100)
|
||
correct, total, confusions = 0, 0, []
|
||
print(f"{'模型':22s} {'own':>6s} {'top1 匹配':22s} {'判定':6s} {'最佳同族':22s} {'同族JSD':>8s} {'间距':>8s}")
|
||
for lname in live:
|
||
valid = {r: v[0] for r, v in results[lname].items() if v[0] is not None}
|
||
if not valid:
|
||
continue
|
||
best = min(valid, key=valid.get)
|
||
own = valid[lname]
|
||
total += 1
|
||
ok = best == lname
|
||
correct += ok
|
||
if not ok:
|
||
confusions.append((lname, best, round(own, 3), round(valid[best], 3)))
|
||
sibs = [r for r in valid if FAMILY[r] == FAMILY[lname] and r != lname]
|
||
if sibs:
|
||
bs = min(sibs, key=valid.get)
|
||
print(f"{lname:22s} {own:6.3f} {best:22s} {'✓' if ok else '✗混淆':6s} "
|
||
f"{bs:22s} {valid[bs]:8.3f} {valid[bs] - own:+8.3f}")
|
||
else:
|
||
print(f"{lname:22s} {own:6.3f} {best:22s} {'✓' if ok else '✗混淆':6s} {'(无同族)':22s}")
|
||
|
||
print()
|
||
print(f"【top-1 正确率】{correct}/{total} = {correct / total * 100:.0f}%")
|
||
print(f"【混淆数】{len(confusions)}")
|
||
for c in confusions:
|
||
print(f" 混淆: {c[0]} 被认成 {c[1]} (own={c[2]}, conf={c[3]})")
|
||
|
||
if "glm_53#rerun" in results:
|
||
valid = {r: v[0] for r, v in results["glm_53#rerun"].items() if v[0] is not None}
|
||
best = min(valid, key=valid.get)
|
||
print(f"【稳定性】glm_53 第二独立样本 top1={best} "
|
||
f"{'✓ 与首跑一致' if best == 'glm_53' else '✗ 不一致'} "
|
||
f"(own={valid['glm_53']:.3f}, 次优={sorted(valid.values())[1]:.3f})")
|