09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
31 lines
1.5 KiB
Python
31 lines
1.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Merge detector 16-cell ref + extra 10-cell ref -> 26-cell fp_fusion ref."""
|
|
import json, sys
|
|
from pathlib import Path
|
|
FP_REFERENCES = Path(__file__).resolve().parent / 'references'
|
|
def main():
|
|
model_key, det_file, extra_file = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
det = json.load(open(FP_REFERENCES / det_file))
|
|
extra = json.load(open(extra_file))
|
|
cells = dict(det.get("cells", {}))
|
|
for cid, c in extra.get("cells", {}).items():
|
|
if cid not in cells:
|
|
cells[cid] = c
|
|
fused = {"formatVersion": 1, "protocol": "one-token/v1",
|
|
"model": det.get("model"), "collectedAt": det.get("collectedAt"),
|
|
"samplesPerCell": det.get("samplesPerCell", 25),
|
|
"postReasoning": det.get("postReasoning", False),
|
|
"meta": {"fusion": True,
|
|
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
|
|
"sourceDetector": det_file, "sourceExtra": extra_file},
|
|
"cells": cells}
|
|
out = FP_REFERENCES / f"{model_key}_fusion_reference.json"
|
|
out.write_text(json.dumps(fused, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
tot_v = sum(c["validCount"] for c in cells.values())
|
|
tot_t = sum(c["totalCount"] for c in cells.values())
|
|
extras = [k for k in sorted(cells) if k.startswith(("binary-", "day-of-week")) and k.endswith(":en")]
|
|
print(f"[{model_key}] {len(cells)} cells -> {out} | valid {tot_v}/{tot_t}")
|
|
print(f" extra_en cells: {extras}")
|
|
if __name__ == "__main__":
|
|
main()
|