Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/deepseek_v4_flash_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

510 lines
13 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "DeepSeek/DeepSeek-V4-Flash",
"collectedAt": "2026-09-01T06:53:23.825Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector cells + 10 fp_fusion-only cells (binary preferences + day-of-week).",
"sourceDetector": "deepseek_v4_flash_reference.json",
"sourceExtra": "/tmp/deepseek_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"5": 1,
"7": 2,
"12": 1,
"17": 1,
"23": 1,
"37": 1,
"42": 6,
"57": 1,
"70": 1,
"73": 9,
"80": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.8022921890824146,
"normalizedEntropy": 0.42178700276434383,
"medianLatencyMs": null,
"meanCompletionTokens": 243.48,
"meanReasoningTokens": 241.24
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"23": 1,
"37": 5,
"42": 14,
"47": 2,
"57": 1,
"73": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.887351814444994,
"normalizedEntropy": 0.28407475425939177,
"medianLatencyMs": null,
"meanCompletionTokens": 54.56,
"meanReasoningTokens": 52.56
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"blue": 22,
"cyan": 1,
"red": 1,
"magenta": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7195563653739032,
"normalizedEntropy": 0.14664202336564808,
"medianLatencyMs": null,
"meanCompletionTokens": 44.64,
"meanReasoningTokens": 42.6
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"giraffe": 5,
"elephant": 13,
"cat": 3,
"penguin": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.7450464172773457,
"normalizedEntropy": 0.309193990527069,
"medianLatencyMs": null,
"meanCompletionTokens": 42.48,
"meanReasoningTokens": 39.32
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"3": 1,
"4": 2,
"5": 1,
"7": 21
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.8743095546405661,
"normalizedEntropy": 0.263193401442427,
"medianLatencyMs": 1554.6371949999884,
"meanCompletionTokens": 73.36,
"meanReasoningTokens": 71.36
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"m": 12,
"g": 2,
"k": 7,
"q": 1,
"x": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.866819311165902,
"normalizedEntropy": 0.3971584411477535,
"medianLatencyMs": 1480.1575740000117,
"meanCompletionTokens": 56.04,
"meanReasoningTokens": 54.04
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"蓝": 23,
"绿": 1,
"紫": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.48217919020227284,
"normalizedEntropy": 0.09826573077333434,
"medianLatencyMs": null,
"meanCompletionTokens": 37.72,
"meanReasoningTokens": 35.72
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 24,
"tails": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.24229218908241482,
"medianLatencyMs": 1675.2963849999942,
"meanCompletionTokens": 73.16,
"meanReasoningTokens": 71.16
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": null,
"meanCompletionTokens": 116.52,
"meanReasoningTokens": 114.52
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"tokyo": 19,
"london": 3,
"cairo": 1,
"kyoto": 1,
"paris": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.225235779471061,
"normalizedEntropy": 0.2170919559734506,
"medianLatencyMs": 1439.2404779999924,
"meanCompletionTokens": 47.64,
"meanReasoningTokens": 45.56
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"5": 3,
"7": 21,
"8": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7641140545540274,
"normalizedEntropy": 0.23002125052918596,
"medianLatencyMs": 1451.3741049999371,
"meanCompletionTokens": 47.16,
"meanReasoningTokens": 45.16
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 1457.2705060000008,
"meanCompletionTokens": 50.92,
"meanReasoningTokens": 48.92
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"k": 8,
"q": 1,
"a": 9,
"m": 6,
"g": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.9222921890824147,
"normalizedEntropy": 0.40896007700373915,
"medianLatencyMs": 1475.0125490000937,
"meanCompletionTokens": 33.8,
"meanReasoningTokens": 31.8
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"熊猫": 6,
"老虎": 2,
"猫": 11,
"大象": 4,
"狗": 1,
"袋鼠": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.1013152774012362,
"normalizedEntropy": 0.37231906815916066,
"medianLatencyMs": null,
"meanCompletionTokens": 35.96,
"meanReasoningTokens": 33.92
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"巴黎": 5,
"东京": 15,
"北京": 2,
"伦敦": 1,
"里斯本": 1,
"上海": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.7553362134321413,
"normalizedEntropy": 0.31101717591819183,
"medianLatencyMs": 1345.1831369999563,
"meanCompletionTokens": 33.56,
"meanReasoningTokens": 31.52
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": null,
"meanCompletionTokens": 37.88,
"meanReasoningTokens": 35.88
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 23,
"winter": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4021791902022728,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"cat": 19,
"dog": 6
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7950402793845223,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"sea": 14,
"mountain": 11
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9895875212220556,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"coffee": 10,
"tea": 15
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"wednesday": 12,
"monday": 13
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9988455359952018,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"wednesday": 20,
"monday": 2,
"tuesday": 1,
"friday": 1,
"thursday": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.106313713864835,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink