09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
337 lines
8.4 KiB
JSON
337 lines
8.4 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "DeepSeek/DeepSeek-V4-Flash-0731",
|
|
"collectedAt": "2026-09-02T06:16:31.339Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"37": 2,
|
|
"42": 15,
|
|
"47": 4,
|
|
"73": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.5797218324096136,
|
|
"normalizedEntropy": 0.23777182818028123,
|
|
"medianLatencyMs": 1697.617889999994,
|
|
"meanCompletionTokens": 60.92,
|
|
"meanReasoningTokens": 58.8
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"7": 1,
|
|
"37": 3,
|
|
"42": 19,
|
|
"47": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.145235779471061,
|
|
"normalizedEntropy": 0.17237516086420482,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 62.12,
|
|
"meanReasoningTokens": 60.12
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"cerulean": 4,
|
|
"blue": 8,
|
|
"purple": 3,
|
|
"magenta": 2,
|
|
"turquoise": 3,
|
|
"teal": 2,
|
|
"chartreuse": 1,
|
|
"indigo": 1,
|
|
"periwinkle": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.8234651896016465,
|
|
"normalizedEntropy": 0.5754082212732725,
|
|
"medianLatencyMs": 1477.725407000049,
|
|
"meanCompletionTokens": 41.92,
|
|
"meanReasoningTokens": 39
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"elephant": 6,
|
|
"platypus": 5,
|
|
"aardvark": 2,
|
|
"giraffe": 6,
|
|
"otter": 1,
|
|
"cat": 3,
|
|
"octopus": 1,
|
|
"cheetah": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.668493070364558,
|
|
"normalizedEntropy": 0.47281379621245656,
|
|
"medianLatencyMs": 1455.671497000003,
|
|
"meanCompletionTokens": 42.88,
|
|
"meanReasoningTokens": 39.4
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 24,
|
|
"8": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 1523.9200239999918,
|
|
"meanCompletionTokens": 41.12,
|
|
"meanReasoningTokens": 39.12
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"q": 13,
|
|
"m": 3,
|
|
"x": 4,
|
|
"k": 3,
|
|
"r": 1,
|
|
"v": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0192365361682794,
|
|
"normalizedEntropy": 0.42958460426056433,
|
|
"medianLatencyMs": 1489.413487999991,
|
|
"meanCompletionTokens": 40.76,
|
|
"meanReasoningTokens": 38.76
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 21,
|
|
"紫": 2,
|
|
"绿": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7943095546405661,
|
|
"normalizedEntropy": 0.16187635309241316,
|
|
"medianLatencyMs": 1400.5907949999964,
|
|
"meanCompletionTokens": 59.28,
|
|
"meanReasoningTokens": 57.28
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1504.2655169999925,
|
|
"meanCompletionTokens": 65.2,
|
|
"meanReasoningTokens": 63.08
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 42.2,
|
|
"meanReasoningTokens": 40.2
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"tokyo": 17,
|
|
"nairobi": 1,
|
|
"paris": 1,
|
|
"quito": 1,
|
|
"kyiv": 2,
|
|
"manila": 1,
|
|
"kyoto": 1,
|
|
"lima": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7843814577244939,
|
|
"normalizedEntropy": 0.31616352325868136,
|
|
"medianLatencyMs": 1433.694755000004,
|
|
"meanCompletionTokens": 45.28,
|
|
"meanReasoningTokens": 43
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"5": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 1517.7847150000016,
|
|
"meanCompletionTokens": 58.96,
|
|
"meanReasoningTokens": 56.96
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 22,
|
|
"tails": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.5293608652873644,
|
|
"normalizedEntropy": 0.5293608652873644,
|
|
"medianLatencyMs": 1477.213821000012,
|
|
"meanCompletionTokens": 41.28,
|
|
"meanReasoningTokens": 39.28
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 10,
|
|
"x": 5,
|
|
"q": 6,
|
|
"z": 1,
|
|
"a": 1,
|
|
"r": 1,
|
|
"e": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.2303083326692295,
|
|
"normalizedEntropy": 0.47448929598256,
|
|
"medianLatencyMs": 1464.9214360000333,
|
|
"meanCompletionTokens": 36.72,
|
|
"meanReasoningTokens": 34.72
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"猫": 21,
|
|
"熊猫": 1,
|
|
"袋鼠": 1,
|
|
"企鹅": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8743095546405661,
|
|
"normalizedEntropy": 0.15491350687223382,
|
|
"medianLatencyMs": 1318.3121069999906,
|
|
"meanCompletionTokens": 35.48,
|
|
"meanReasoningTokens": 33.36
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"巴黎": 5,
|
|
"东京": 10,
|
|
"北京": 7,
|
|
"上海": 2,
|
|
"里约热内卢": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.984639954666178,
|
|
"normalizedEntropy": 0.3516460887614139,
|
|
"medianLatencyMs": 1544.7924609999754,
|
|
"meanCompletionTokens": 52.88,
|
|
"meanReasoningTokens": 50.72
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"5": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.018234106192829464,
|
|
"medianLatencyMs": 1460.929415000006,
|
|
"meanCompletionTokens": 36.72,
|
|
"meanReasoningTokens": 34.72
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
} |