Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/kimi_k2_6_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

556 lines
14 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "MoonshotAi/Kimi-K2.6",
"collectedAt": "2026-09-04T09:39:19.021Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
"sourceDetector": "kimi_k26_tmp_reference.json",
"sourceExtra": "/tmp/bfd/kimi_k26_tmp_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"23": 1,
"37": 2,
"42": 3,
"47": 3,
"56": 1,
"57": 3,
"58": 1,
"67": 1,
"73": 8,
"77": 1,
"84": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.033269689515108,
"normalizedEntropy": 0.4565525807412093,
"medianLatencyMs": 3640.9765949249268,
"meanCompletionTokens": 143.56,
"meanReasoningTokens": 140.72
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"37": 1,
"42": 7,
"56": 1,
"67": 2,
"73": 14
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.6456780552463373,
"normalizedEntropy": 0.24769922891755697,
"medianLatencyMs": 2893.133331298828,
"meanCompletionTokens": 82.24,
"meanReasoningTokens": 79.28
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"indigo": 8,
"magenta": 2,
"violet": 2,
"crimson": 2,
"turquoise": 1,
"cerulean": 2,
"teal": 3,
"amber": 2,
"cyan": 2,
"azure": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.0136606896881855,
"normalizedEntropy": 0.6141691221698111,
"medianLatencyMs": 4363.527551651001,
"meanCompletionTokens": 157.56,
"meanReasoningTokens": 153.6
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"platypus": 5,
"giraffe": 2,
"tiger": 1,
"otter": 1,
"axolotl": 2,
"octopus": 7,
"tapir": 1,
"penguin": 3,
"badger": 1,
"elephant": 1,
"narwhal": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.043215692534584,
"normalizedEntropy": 0.5392085818997551,
"medianLatencyMs": 5141.708791732788,
"meanCompletionTokens": 260.28,
"meanReasoningTokens": 256.16
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"3": 2,
"4": 1,
"7": 22
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6395563653739031,
"normalizedEntropy": 0.19252564989537765,
"medianLatencyMs": 2346.886293411255,
"meanCompletionTokens": 125.56,
"meanReasoningTokens": 122.64
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"x": 2,
"q": 9,
"z": 3,
"n": 2,
"k": 4,
"w": 1,
"j": 2,
"m": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.6724876891689533,
"normalizedEntropy": 0.5685612090406419,
"medianLatencyMs": 3526.5053329467773,
"meanCompletionTokens": 123.32,
"meanReasoningTokens": 120.48
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"绯红": 2,
"靛蓝": 3,
"蓝": 6,
"青": 2,
"紫": 5,
"碧": 1,
"靛": 2,
"绛紫": 2,
"琥珀": 1,
"翠绿": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.0488840705376354,
"normalizedEntropy": 0.6213474727287116,
"medianLatencyMs": 5116.957455635071,
"meanCompletionTokens": 182.44,
"meanReasoningTokens": 178.96
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 24,
"tails": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.24229218908241482,
"medianLatencyMs": 3748.6917638778687,
"meanCompletionTokens": 130.36,
"meanReasoningTokens": 127.04
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 9769.422966957092,
"meanCompletionTokens": 362.76,
"meanReasoningTokens": 359.92
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"oslo": 3,
"glasgow": 1,
"paris": 3,
"adelaide": 1,
"tbilisi": 1,
"osaka": 2,
"mumbai": 1,
"hanoi": 1,
"lisbon": 6,
"dakar": 1,
"hamburg": 1,
"kyoto": 1,
"nairobi": 1,
"lima": 1,
"rotterdam": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.5630741894285687,
"normalizedEntropy": 0.6313190963093603,
"medianLatencyMs": 5643.602588653564,
"meanCompletionTokens": 203.4,
"meanReasoningTokens": 199.4
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 2319.5310754776,
"meanCompletionTokens": 72.28,
"meanReasoningTokens": 69.32
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 19,
"tails": 6
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7950402793845223,
"normalizedEntropy": 0.7950402793845223,
"medianLatencyMs": 2901.511685371399,
"meanCompletionTokens": 95.76,
"meanReasoningTokens": 92.76
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"m": 5,
"k": 10,
"x": 2,
"q": 2,
"r": 4,
"j": 1,
"w": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.370699332842307,
"normalizedEntropy": 0.5043569272237918,
"medianLatencyMs": 2935.3015670776367,
"meanCompletionTokens": 91.52,
"meanReasoningTokens": 88.56
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"虎": 3,
"鹤": 1,
"猫": 7,
"刺猬": 1,
"企鹅": 4,
"鲸": 2,
"水豚": 1,
"大象": 1,
"长颈鹿": 1,
"豹": 1,
"海豚": 1,
"海獭": 1,
"树懒": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.2676013115120557,
"normalizedEntropy": 0.5789660830536653,
"medianLatencyMs": 5722.084982872009,
"meanCompletionTokens": 297.48,
"meanReasoningTokens": 294.52
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"泉州": 2,
"杭州": 11,
"珀斯": 1,
"贵阳": 1,
"敦煌": 1,
"伊斯坦布尔": 1,
"洛阳": 1,
"布拉格": 1,
"鹿特丹": 1,
"成都": 1,
"京都": 2,
"重庆": 1
},
"validCount": 24,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 1,
"totalCount": 25,
"entropyBits": 2.8327230088457287,
"normalizedEntropy": 0.501912684093178,
"medianLatencyMs": 3940.9336037635803,
"meanCompletionTokens": 117.5,
"meanReasoningTokens": 114.5
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 4330.780131340027,
"meanCompletionTokens": 143.44,
"meanReasoningTokens": 140.6
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"winter": 5,
"summer": 20
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7219280948873623,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"dog": 23,
"cat": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4021791902022728,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"mountain": 22,
"sea": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.5293608652873644,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"coffee": 16,
"tea": 9
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9426831892554922,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"thursday": 14,
"wednesday": 6,
"tuesday": 2,
"friday": 1,
"saturday": 1,
"sunday": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.811346433249389,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"wednesday": 15,
"thursday": 10
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink