09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
54 lines
1.9 KiB
Python
54 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
||
"""文本探针的模型内复测 vs 跨模型区分度（纯离线）。
|
||
|
||
关键问题：s_joke 跨模型 Jaccard 0.06 是"风格指纹"还是"纯随机"？
|
||
若同一模型两次独立采样（glm_53 verify vs rerun）的 Jaccard 同样趋近 0，
|
||
则区分度被随机性淹没（模型内不稳定 → 无法建参考 → 打分端无法消费）。
|
||
对比组：I 层身份题（temp 0.2，预期模型内近乎逐字稳定）。
|
||
"""
|
||
import itertools
|
||
import json
|
||
import re
|
||
|
||
BFD = "/tmp/bfd"
|
||
DIRS = ["deepseek_v4_flash", "deepseek_v4_flash_0731", "deepseek_v4_pro",
|
||
"glm_51", "glm_52", "glm_53", "kimi_k2_6", "kimi_k2_7code", "kimi_k3"]
|
||
|
||
|
||
def toks(t):
|
||
return set(re.findall(r"\w+", (t or "").lower()))
|
||
|
||
|
||
def jac(a, b):
|
||
return len(a & b) / len(a | b) if (a or b) else 1.0
|
||
|
||
|
||
def load(path):
|
||
out = {}
|
||
for r in map(json.loads, open(path)):
|
||
if not r.get("error"):
|
||
out[r["id"]] = r.get("response") or ""
|
||
return out
|
||
|
||
|
||
r1 = {d: load(f"{BFD}/{d}/raw_answers.jsonl") for d in DIRS}
|
||
r2 = load(f"{BFD}/glm_53/rerun/raw_answers.jsonl")
|
||
|
||
PROBES = ["s_joke", "s_list", "s_simple", "s_what", "s_len1a",
|
||
"i_zh_direct", "i_direct_en1", "i_meta1", "k_cutoff1", "k_params"]
|
||
|
||
print(f"{'探针':18s} {'模型内复测(g53两跑)':>20s} {'逐字相同':>8s} {'跨模型均J':>10s} 判读")
|
||
for p in PROBES:
|
||
within = jac(toks(r1["glm_53"].get(p, "")), toks(r2.get(p, "")))
|
||
verbatim = r1["glm_53"].get(p, "") == r2.get(p, "")
|
||
ts = [toks(r1[d].get(p)) for d in DIRS if r1[d].get(p)]
|
||
cross = sum(jac(a, b) for a, b in itertools.combinations(ts, 2)) / \
|
||
max(len(list(itertools.combinations(ts, 2))), 1)
|
||
if within > 0.6 and cross < 0.5:
|
||
verdict = "真指纹: 模型内稳+模型间异"
|
||
elif within < 0.3:
|
||
verdict = "纯随机: 模型内也不稳→不可建参考"
|
||
else:
|
||
verdict = "部分信号"
|
||
print(f"{p:18s} {within:20.2f} {str(verbatim):>8s} {cross:10.2f} {verdict}")
|