09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
522 lines
13 KiB
JSON
522 lines
13 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "MoonshotAi/Kimi-K2.7-Code",
|
|
"collectedAt": "2026-09-04T09:56:49.375Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"meta": {
|
|
"fusion": true,
|
|
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
|
|
"sourceDetector": "kimi_k27code_tmp_reference.json",
|
|
"sourceExtra": "/tmp/bfd/kimi_k27code_tmp_extra_cells.json"
|
|
},
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"37": 2,
|
|
"42": 8,
|
|
"47": 10,
|
|
"73": 5
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.810699332842307,
|
|
"normalizedEntropy": 0.27253740615714667,
|
|
"medianLatencyMs": 1929.3976545333862,
|
|
"meanCompletionTokens": 49.88,
|
|
"meanReasoningTokens": 46.88
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 2,
|
|
"42": 4,
|
|
"47": 1,
|
|
"73": 18
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2415101887362598,
|
|
"normalizedEntropy": 0.1868659033660324,
|
|
"medianLatencyMs": 4234.577438354492,
|
|
"meanCompletionTokens": 56.88,
|
|
"meanReasoningTokens": 53.88
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"magenta": 8,
|
|
"azure": 5,
|
|
"vermilion": 1,
|
|
"indigo": 4,
|
|
"cerulean": 1,
|
|
"cyan": 2,
|
|
"violet": 1,
|
|
"teal": 2,
|
|
"crimson": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7394705707972515,
|
|
"normalizedEntropy": 0.5582905339786818,
|
|
"medianLatencyMs": 1602.4216299057007,
|
|
"meanCompletionTokens": 44.2,
|
|
"meanReasoningTokens": 40.24
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"okapi": 1,
|
|
"octopus": 7,
|
|
"platypus": 4,
|
|
"elephant": 5,
|
|
"otter": 2,
|
|
"penguin": 4,
|
|
"pangolin": 1,
|
|
"tiger": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.673411192621123,
|
|
"normalizedEntropy": 0.47368520790176843,
|
|
"medianLatencyMs": 2826.1588563919067,
|
|
"meanCompletionTokens": 42.04,
|
|
"meanReasoningTokens": 37.76
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"6": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 1835.6529273986816,
|
|
"meanCompletionTokens": 55.8,
|
|
"meanReasoningTokens": 52.8
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"q": 18,
|
|
"m": 6,
|
|
"k": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.0211191885631823,
|
|
"normalizedEntropy": 0.21723907757442953,
|
|
"medianLatencyMs": 2027.0736074447632,
|
|
"meanCompletionTokens": 43,
|
|
"meanReasoningTokens": 40
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"靛蓝": 2,
|
|
"蓝": 14,
|
|
"紫": 6,
|
|
"橙": 2,
|
|
"青": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7313464332493889,
|
|
"normalizedEntropy": 0.3528398278940391,
|
|
"medianLatencyMs": 1975.8200035095215,
|
|
"meanCompletionTokens": 69.88,
|
|
"meanReasoningTokens": 66.72
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1818.241452217102,
|
|
"meanCompletionTokens": 51.68,
|
|
"meanReasoningTokens": 47.92
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 2659.0778017044067,
|
|
"meanCompletionTokens": 63.36,
|
|
"meanReasoningTokens": 60.36
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"kyoto": 4,
|
|
"osaka": 2,
|
|
"lisbon": 8,
|
|
"paris": 2,
|
|
"tokyo": 2,
|
|
"budapest": 2,
|
|
"tbilisi": 1,
|
|
"timbuktu": 2,
|
|
"mumbai": 1,
|
|
"baku": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.963856189774724,
|
|
"normalizedEntropy": 0.5251473620367048,
|
|
"medianLatencyMs": 2277.6057929992676,
|
|
"meanCompletionTokens": 46.44,
|
|
"meanReasoningTokens": 41.92
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"3": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 2840.0440530776978,
|
|
"meanCompletionTokens": 70.4,
|
|
"meanReasoningTokens": 67.4
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 19,
|
|
"tails": 6
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7950402793845223,
|
|
"normalizedEntropy": 0.7950402793845223,
|
|
"medianLatencyMs": 2722.0782718658447,
|
|
"meanCompletionTokens": 58.96,
|
|
"meanReasoningTokens": 55.96
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"x": 3,
|
|
"k": 6,
|
|
"m": 9,
|
|
"q": 6,
|
|
"l": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0717056888227985,
|
|
"normalizedEntropy": 0.44074720942110224,
|
|
"medianLatencyMs": 1683.4700717926025,
|
|
"meanCompletionTokens": 47.4,
|
|
"meanReasoningTokens": 44.4
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"长颈鹿": 5,
|
|
"海豚": 1,
|
|
"企鹅": 5,
|
|
"老虎": 4,
|
|
"熊猫": 3,
|
|
"猫": 5,
|
|
"树懒": 1,
|
|
"大象": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7405038327557683,
|
|
"normalizedEntropy": 0.48557293818380515,
|
|
"medianLatencyMs": 2110.104063987732,
|
|
"meanCompletionTokens": 69.04,
|
|
"meanReasoningTokens": 66
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"成都": 5,
|
|
"杭州": 7,
|
|
"北京": 3,
|
|
"京都": 2,
|
|
"西安": 1,
|
|
"上海": 1,
|
|
"桂林": 1,
|
|
"墨尔本": 1,
|
|
"喀什": 1,
|
|
"雷克雅未克": 1,
|
|
"拉萨": 1,
|
|
"苏州": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 3.123215692534583,
|
|
"normalizedEntropy": 0.5533832875105995,
|
|
"medianLatencyMs": 3302.0929412841797,
|
|
"meanCompletionTokens": 85.04,
|
|
"meanReasoningTokens": 81.88
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 24,
|
|
"42": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.018234106192829464,
|
|
"medianLatencyMs": 3258.5312995910645,
|
|
"meanCompletionTokens": 86.6,
|
|
"meanReasoningTokens": 83.6
|
|
},
|
|
"binary-season:en": {
|
|
"cellId": "binary-season:en",
|
|
"counts": {
|
|
"summer": 24,
|
|
"winter": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-season:zh": {
|
|
"cellId": "binary-season:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:en": {
|
|
"cellId": "binary-pet:en",
|
|
"counts": {
|
|
"dog": 22,
|
|
"cat": 2
|
|
},
|
|
"validCount": 24,
|
|
"invalidCount": 1,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4538021177829141,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:zh": {
|
|
"cellId": "binary-pet:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:en": {
|
|
"cellId": "binary-sea-mountain:en",
|
|
"counts": {
|
|
"mountain": 20,
|
|
"sea": 5
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7219280948873623,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:zh": {
|
|
"cellId": "binary-sea-mountain:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:en": {
|
|
"cellId": "binary-tea-coffee:en",
|
|
"counts": {
|
|
"coffee": 15,
|
|
"tea": 10
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9709505944546686,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:zh": {
|
|
"cellId": "binary-tea-coffee:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:en": {
|
|
"cellId": "day-of-week:en",
|
|
"counts": {
|
|
"wednesday": 7,
|
|
"saturday": 1,
|
|
"tuesday": 13,
|
|
"thursday": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.6135681581652277,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:zh": {
|
|
"cellId": "day-of-week:zh",
|
|
"counts": {
|
|
"wednesday": 16,
|
|
"thursday": 7,
|
|
"tuesday": 1,
|
|
"monday": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2977968115985956,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
}
|
|
}
|
|
} |