09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
324 lines
7.9 KiB
JSON
324 lines
7.9 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "Qwen3-4B",
|
|
"collectedAt": "2026-08-21T06:51:26.314Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"42": 20,
|
|
"50": 5
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7219280948873623,
|
|
"normalizedEntropy": 0.10866100563682445,
|
|
"medianLatencyMs": 5752.167354000005,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"42": 20,
|
|
"50": 1,
|
|
"57": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8663137138648347,
|
|
"normalizedEntropy": 0.13039320676418933,
|
|
"medianLatencyMs": 5856.288877999992,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"blue": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5261.671336999978,
|
|
"meanCompletionTokens": 4.08,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"rabbit": 11,
|
|
"dog": 3,
|
|
"zebra": 8,
|
|
"cat": 2,
|
|
"bear": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.891510777487775,
|
|
"normalizedEntropy": 0.33514510538286324,
|
|
"medianLatencyMs": 5708.354767999961,
|
|
"meanCompletionTokens": 5,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5508.977116000024,
|
|
"meanCompletionTokens": 1,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"x": 16,
|
|
"r": 3,
|
|
"m": 3,
|
|
"b": 1,
|
|
"k": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.6234651896016472,
|
|
"normalizedEntropy": 0.34538581216901293,
|
|
"medianLatencyMs": 5168.777773000009,
|
|
"meanCompletionTokens": 1,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 21,
|
|
"蓝紫": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6343095546405662,
|
|
"normalizedEntropy": 0.12926914555793217,
|
|
"medianLatencyMs": 5445.874789000023,
|
|
"meanCompletionTokens": 2.12,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5668.764903000003,
|
|
"meanCompletionTokens": 5,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5405.636815999984,
|
|
"meanCompletionTokens": 1.16,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"paris": 10,
|
|
"chicago": 4,
|
|
"cairo": 1,
|
|
"los": 2,
|
|
"new": 3,
|
|
"dallas": 1,
|
|
"denver": 1,
|
|
"rome": 1,
|
|
"oklahoma": 1,
|
|
"austin": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7248894517332403,
|
|
"normalizedEntropy": 0.48280632250518146,
|
|
"medianLatencyMs": 5475.599871000042,
|
|
"meanCompletionTokens": 6.56,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5423.345439999946,
|
|
"meanCompletionTokens": 1,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5572.728058000008,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"b": 4,
|
|
"x": 16,
|
|
"k": 1,
|
|
"r": 3,
|
|
"m": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.5736606896881862,
|
|
"normalizedEntropy": 0.3347901013632253,
|
|
"medianLatencyMs": 5144.88217300002,
|
|
"meanCompletionTokens": 1,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"狐狸": 4,
|
|
"狮子": 5,
|
|
"企鹅": 4,
|
|
"老虎": 5,
|
|
"熊猫": 1,
|
|
"兔子": 1,
|
|
"猫": 4,
|
|
"猴子": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7550849518197795,
|
|
"normalizedEntropy": 0.4881564765614181,
|
|
"medianLatencyMs": 5298.365481000044,
|
|
"meanCompletionTokens": 1.84,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"上海": 17,
|
|
"北京": 2,
|
|
"杭州": 2,
|
|
"广州": 1,
|
|
"西安": 1,
|
|
"巴黎": 1,
|
|
"成都": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.704381457724494,
|
|
"normalizedEntropy": 0.30198881764783675,
|
|
"medianLatencyMs": 5206.236279000004,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5461.653563999978,
|
|
"meanCompletionTokens": 1,
|
|
"meanReasoningTokens": null
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
}
|