Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/kimi_k3_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

507 lines
13 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "MoonshotAi/Kimi-K3",
"collectedAt": "2026-09-01T08:25:58.034Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector cells + 10 fp_fusion-only cells.",
"sourceDetector": "kimi_k3_reference.json",
"sourceExtra": "/tmp/kimi_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"37": 6,
"42": 9,
"47": 7,
"73": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.9060373108197468,
"normalizedEntropy": 0.2868872017057274,
"medianLatencyMs": 4201.567929000012,
"meanCompletionTokens": 54.92,
"meanReasoningTokens": 40.52
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"37": 7,
"42": 4,
"47": 9,
"57": 4,
"73": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.0766238110793633,
"normalizedEntropy": 0.31256302842247047,
"medianLatencyMs": 4935.542820999981,
"meanCompletionTokens": 47.76,
"meanReasoningTokens": 33.76
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"coral": 1,
"crimson": 3,
"blue": 7,
"chartreuse": 2,
"cerulean": 2,
"azure": 8,
"teal": 1,
"indigo": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.5476013115120564,
"normalizedEntropy": 0.5191885292474349,
"medianLatencyMs": 4126.816009999951,
"meanCompletionTokens": 31.76,
"meanReasoningTokens": 16.44
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"otter": 17,
"elephant": 2,
"penguin": 1,
"capybara": 1,
"pangolin": 1,
"octopus": 2,
"platypus": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.704381457724494,
"normalizedEntropy": 0.30198881764783675,
"medianLatencyMs": 4583.067276999936,
"meanCompletionTokens": 26,
"meanReasoningTokens": 10.88
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 4282.95300400001,
"meanCompletionTokens": 40.88,
"meanReasoningTokens": 25.92
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"m": 4,
"q": 19,
"k": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.0154312795575997,
"normalizedEntropy": 0.2160289973805212,
"medianLatencyMs": 4654.510852000036,
"meanCompletionTokens": 38,
"meanReasoningTokens": 23.72
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"蓝": 23,
"蔚蓝": 1,
"紫": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.48217919020227284,
"normalizedEntropy": 0.09826573077333434,
"medianLatencyMs": 3668.3515180000104,
"meanCompletionTokens": 40.52,
"meanReasoningTokens": 28.32
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 23,
"tails": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4021791902022728,
"normalizedEntropy": 0.4021791902022728,
"medianLatencyMs": 5788.457129999995,
"meanCompletionTokens": 57.12,
"meanReasoningTokens": 42.28
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 21,
"42": 2
},
"validCount": 23,
"invalidCount": 2,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4262286569981449,
"normalizedEntropy": 0.032076554442651374,
"medianLatencyMs": 4735.351004000055,
"meanCompletionTokens": 69.8,
"meanReasoningTokens": 49.6
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"timbuktu": 2,
"tokyo": 5,
"lisbon": 11,
"osaka": 1,
"reykjavik": 2,
"kyoto": 3,
"tucson": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.3071251585103023,
"normalizedEntropy": 0.40878524911570996,
"medianLatencyMs": 4120.0932230000035,
"meanCompletionTokens": 34.16,
"meanReasoningTokens": 18.08
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 5510.148357999977,
"meanCompletionTokens": 52.8,
"meanReasoningTokens": 37.36
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 18,
"tails": 7
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.8554508105601306,
"normalizedEntropy": 0.8554508105601306,
"medianLatencyMs": 3399.379054000019,
"meanCompletionTokens": 62.36,
"meanReasoningTokens": 47.28
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"k": 4,
"q": 16,
"m": 5
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.2994705707972523,
"normalizedEntropy": 0.2764572356458516,
"medianLatencyMs": 4990.088311000029,
"meanCompletionTokens": 49.76,
"meanReasoningTokens": 34.84
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"水獭": 2,
"水豚": 2,
"熊猫": 8,
"猫": 11,
"海豚": 1,
"狐狸": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.0017062775743137,
"normalizedEntropy": 0.35466996504994436,
"medianLatencyMs": 5375.511597000004,
"meanCompletionTokens": 51.52,
"meanReasoningTokens": 34.84
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"杭州": 1,
"西安": 4,
"昆明": 4,
"北京": 3,
"成都": 5,
"巴黎": 5,
"雷克雅未克": 1,
"青岛": 1,
"维也纳": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.8848894517332404,
"normalizedEntropy": 0.5111557337268707,
"medianLatencyMs": 4517.09676100011,
"meanCompletionTokens": 48.16,
"meanReasoningTokens": 34.12
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"7": 22
},
"validCount": 22,
"invalidCount": 3,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 4412.280236000079,
"meanCompletionTokens": 64.36,
"meanReasoningTokens": 41.2
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 24,
"winter": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"dog": 15,
"cat": 10
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"mountain": 10,
"sea": 15
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"coffee": 21,
"tea": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6343095546405662,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"wednesday": 16,
"thursday": 1,
"tuesday": 8
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.1238561897747248,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"wednesday": 17,
"thursday": 5,
"monday": 2,
"friday": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.3199958387470214,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink