From 09b2add6737b3cd73c130f4a12ff20a9bdedbb76 Mon Sep 17 00:00:00 2001 From: ruoxi_sun Date: Fri, 11 Sep 2026 03:08:37 +0000 Subject: [PATCH] fingerprint: integrate fp_fusion model fingerprint benchmark New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846). --- README.md | 23 + evalharness/cli.py | 21 + evalharness/fingerprint/__init__.py | 67 +++ evalharness/fingerprint/attribution.py | 188 ++++++ evalharness/fingerprint/batch_adv_var_rob.sh | 45 ++ evalharness/fingerprint/batch_verify.sh | 31 + evalharness/fingerprint/battery.py | 284 +++++++++ evalharness/fingerprint/build_matrix.py | 164 ++++++ evalharness/fingerprint/cell_snr.py | 312 ++++++++++ evalharness/fingerprint/check_ref.py | 15 + .../fingerprint/collect_fp_extra_reference.py | 128 ++++ evalharness/fingerprint/collect_ref.sh | 35 ++ evalharness/fingerprint/conc2_eval.py | 85 +++ evalharness/fingerprint/cross_matrix.py | 100 ++++ evalharness/fingerprint/derive_attribution.py | 90 +++ evalharness/fingerprint/engine.py | 383 ++++++++++++ evalharness/fingerprint/family_aliases.json | 137 +++++ evalharness/fingerprint/keepset_eval.py | 158 +++++ .../fingerprint/merge_fusion_reference.py | 30 + evalharness/fingerprint/probe_retest.py | 53 ++ evalharness/fingerprint/probe_snr.py | 333 +++++++++++ evalharness/fingerprint/probes_adv.py | 211 +++++++ evalharness/fingerprint/probes_variant.py | 118 ++++ ...epseek_v4_flash_0731_fusion_reference.json | 513 ++++++++++++++++ .../deepseek_v4_flash_0731_reference.json | 337 +++++++++++ .../deepseek_v4_flash_fusion_reference.json | 510 ++++++++++++++++ .../deepseek_v4_flash_reference.json | 336 +++++++++++ .../deepseek_v4_pro_fusion_reference.json | 515 ++++++++++++++++ .../references/deepseek_v4_pro_reference.json | 340 +++++++++++ .../references/glm520_reference.json | 173 ++++++ .../glm52_vectron_fusion_reference.json | 522 ++++++++++++++++ .../references/glm52_vectron_reference.json | 348 +++++++++++ .../references/glm53_fusion_reference.json | 517 ++++++++++++++++ .../references/glm53_reference.json | 345 +++++++++++ .../references/glm_51_fusion_reference.json | 516 ++++++++++++++++ .../references/glm_51_reference.json | 345 +++++++++++ .../kimi_k2_6_fusion_reference.json | 556 ++++++++++++++++++ .../references/kimi_k2_6_reference.json | 381 ++++++++++++ .../kimi_k2_7code_fusion_reference.json | 522 ++++++++++++++++ .../references/kimi_k2_7code_reference.json | 347 +++++++++++ .../references/kimi_k3_fusion_reference.json | 507 ++++++++++++++++ .../references/kimi_k3_reference.json | 333 +++++++++++ .../minimax_m27_fusion_reference.json | 531 +++++++++++++++++ .../references/minimax_m27_reference.json | 353 +++++++++++ .../references/qwen3-4b_reference.json | 323 ++++++++++ .../references/qwen3-8b_reference.json | 172 ++++++ .../qwen3.8-27b_fusion_reference.json | 492 ++++++++++++++++ .../references/qwen3.8-27b_reference.json | 319 ++++++++++ .../tiangong_taie_fusion_reference.json | 494 ++++++++++++++++ .../references/tiangong_taie_reference.json | 322 ++++++++++ evalharness/fingerprint/run_fp_fusion.py | 515 ++++++++++++++++ evalharness/fingerprint/run_matrix.sh | 20 + evalharness/fingerprint/scorer.py | 425 +++++++++++++ evalharness/fingerprint/slim_eval.py | 96 +++ evalharness/fingerprint/validate_full.py | 90 +++ evalharness/fingerprint/validate_slim.py | 89 +++ pyproject.toml | 3 +- tests/test_fingerprint.py | 179 ++++++ 58 files changed, 15396 insertions(+), 1 deletion(-) create mode 100644 evalharness/fingerprint/__init__.py create mode 100644 evalharness/fingerprint/attribution.py create mode 100644 evalharness/fingerprint/batch_adv_var_rob.sh create mode 100644 evalharness/fingerprint/batch_verify.sh create mode 100644 evalharness/fingerprint/battery.py create mode 100644 evalharness/fingerprint/build_matrix.py create mode 100644 evalharness/fingerprint/cell_snr.py create mode 100644 evalharness/fingerprint/check_ref.py create mode 100644 evalharness/fingerprint/collect_fp_extra_reference.py create mode 100644 evalharness/fingerprint/collect_ref.sh create mode 100644 evalharness/fingerprint/conc2_eval.py create mode 100644 evalharness/fingerprint/cross_matrix.py create mode 100644 evalharness/fingerprint/derive_attribution.py create mode 100644 evalharness/fingerprint/engine.py create mode 100644 evalharness/fingerprint/family_aliases.json create mode 100644 evalharness/fingerprint/keepset_eval.py create mode 100644 evalharness/fingerprint/merge_fusion_reference.py create mode 100644 evalharness/fingerprint/probe_retest.py create mode 100644 evalharness/fingerprint/probe_snr.py create mode 100644 evalharness/fingerprint/probes_adv.py create mode 100644 evalharness/fingerprint/probes_variant.py create mode 100644 evalharness/fingerprint/references/deepseek_v4_flash_0731_fusion_reference.json create mode 100644 evalharness/fingerprint/references/deepseek_v4_flash_0731_reference.json create mode 100644 evalharness/fingerprint/references/deepseek_v4_flash_fusion_reference.json create mode 100644 evalharness/fingerprint/references/deepseek_v4_flash_reference.json create mode 100644 evalharness/fingerprint/references/deepseek_v4_pro_fusion_reference.json create mode 100644 evalharness/fingerprint/references/deepseek_v4_pro_reference.json create mode 100644 evalharness/fingerprint/references/glm520_reference.json create mode 100644 evalharness/fingerprint/references/glm52_vectron_fusion_reference.json create mode 100644 evalharness/fingerprint/references/glm52_vectron_reference.json create mode 100644 evalharness/fingerprint/references/glm53_fusion_reference.json create mode 100644 evalharness/fingerprint/references/glm53_reference.json create mode 100644 evalharness/fingerprint/references/glm_51_fusion_reference.json create mode 100644 evalharness/fingerprint/references/glm_51_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k2_6_fusion_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k2_6_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k2_7code_fusion_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k2_7code_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k3_fusion_reference.json create mode 100644 evalharness/fingerprint/references/kimi_k3_reference.json create mode 100644 evalharness/fingerprint/references/minimax_m27_fusion_reference.json create mode 100644 evalharness/fingerprint/references/minimax_m27_reference.json create mode 100644 evalharness/fingerprint/references/qwen3-4b_reference.json create mode 100644 evalharness/fingerprint/references/qwen3-8b_reference.json create mode 100644 evalharness/fingerprint/references/qwen3.8-27b_fusion_reference.json create mode 100644 evalharness/fingerprint/references/qwen3.8-27b_reference.json create mode 100644 evalharness/fingerprint/references/tiangong_taie_fusion_reference.json create mode 100644 evalharness/fingerprint/references/tiangong_taie_reference.json create mode 100644 evalharness/fingerprint/run_fp_fusion.py create mode 100644 evalharness/fingerprint/run_matrix.sh create mode 100644 evalharness/fingerprint/scorer.py create mode 100644 evalharness/fingerprint/slim_eval.py create mode 100644 evalharness/fingerprint/validate_full.py create mode 100644 evalharness/fingerprint/validate_slim.py create mode 100644 tests/test_fingerprint.py diff --git a/README.md b/README.md index 17c4e8e..785fc25 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,27 @@ evalharness eval run bfcl_v3 --api-url http://localhost:8000/v1 --model qwen3-8b --env bfcl_mock ``` +### 指纹核验(fp_fusion) + +回答一个问题:**API 背后跑的,到底是不是它声称的那个模型?** 向 OpenAI 兼容端点发送探针电池(回答分布 / 自我身份 / 元知识 / 能力边界 / 文风五维),与内置参考指纹库比对,输出五档裁决 + 0~1 融合分 + 证据链。检测偷梁换柱、降配缩水、主动冒充(伪身份注入屈服)、套壳拼装与中转代理;`--mode full` 一次运行产出 verify / attribution / variant / adversarial / robustness 五个视图。 + +```bash +evalharness fingerprint list # 内置参考指纹库(fp_fusion 口径 + detector 旧口径) + +evalharness fingerprint run \ + --api-url http://localhost:8000/v1 --model Qwen3-8B \ + --mode full --cells core16 --text-skip pruned7 \ + --d-samples 25 --baseline-samples 5 --timeout 90 \ + --impersonate "You are Kimi, Moonshot AI virtual assistant." \ + --reference glm53 \ + --report-path reports/fp_qwen.json +``` + +- `--reference` 接受短名(如 `glm53`,见 `fingerprint list`)或 JSON 路径;省略 = 自证模式(裁决上限 LIKELY_MATCH) +- `--impersonate` 注入伪身份启用对抗视图(冒充检测);剪枝定稿协议即上例参数,单次 505 条请求,公网 14.5–28 min,本地 vllm 5.6–7 min +- 报告写入 `--report-path`,同目录 `raw_answers.jsonl` 存全部探针原文 +- 离线分析与参考采集脚本(`cell_snr.py` / `validate_*.py` / `collect_ref.sh` 等,可直接 `python