Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/qwen3.8-27b_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

492 lines
12 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "Qwen3.8-27B",
"collectedAt": "2026-09-03T10:20:36.636Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
"sourceDetector": "qwen3.8-27b_reference.json",
"sourceExtra": "qwen_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"42": 23,
"47": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4021791902022728,
"normalizedEntropy": 0.06053399994136682,
"medianLatencyMs": 182.66270351409912,
"meanCompletionTokens": 3,
"meanReasoningTokens": null
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"7": 2,
"42": 20,
"47": 2,
"73": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.0263137138648348,
"normalizedEntropy": 0.15447560641730784,
"medianLatencyMs": 178.68310260772705,
"meanCompletionTokens": 2.92,
"meanReasoningTokens": null
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"blue": 14,
"purple": 1,
"cobalt": 2,
"teal": 6,
"turquoise": 1,
"indigo": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.811346433249389,
"normalizedEntropy": 0.3691434316612796,
"medianLatencyMs": 143.06485271453857,
"meanCompletionTokens": 2.48,
"meanReasoningTokens": null
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"kangaroo": 2,
"platypus": 5,
"koala": 1,
"ostrich": 7,
"falcon": 1,
"otter": 4,
"fox": 4,
"elephant": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.673411192621123,
"normalizedEntropy": 0.47368520790176843,
"medianLatencyMs": 190.27048015594482,
"meanCompletionTokens": 3.6,
"meanReasoningTokens": null
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 123.02077293395996,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"q": 16,
"k": 9
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9426831892554922,
"normalizedEntropy": 0.20055212826520413,
"medianLatencyMs": 122.49901103973389,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"蓝": 22,
"红": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.5293608652873644,
"normalizedEntropy": 0.10788112246910952,
"medianLatencyMs": 127.85008716583252,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 180.7693395614624,
"meanCompletionTokens": 3,
"meanReasoningTokens": null
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 126.00289821624756,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"kyoto": 11,
"paris": 2,
"lisbon": 2,
"osaka": 4,
"tokyo": 3,
"oslo": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.2613152774012364,
"normalizedEntropy": 0.40066847938084993,
"medianLatencyMs": 181.48858451843262,
"meanCompletionTokens": 3,
"meanReasoningTokens": null
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 127.78436660766602,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 22
},
"validCount": 22,
"invalidCount": 3,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 99.95673847198486,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"q": 13,
"k": 11,
"m": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.1974776241409462,
"normalizedEntropy": 0.2547586387544438,
"medianLatencyMs": 127.07363319396973,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"猫": 14,
"狐狸": 2,
"狐": 5,
"熊猫": 1,
"鲸": 1,
"海豚": 1,
"鲸鱼": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.967351814444994,
"normalizedEntropy": 0.34858291003398534,
"medianLatencyMs": 139.31216621398926,
"meanCompletionTokens": 2.04,
"meanReasoningTokens": null
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"巴黎": 11,
"东京": 13,
"成都": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.1974776241409462,
"normalizedEntropy": 0.21217365997214463,
"medianLatencyMs": 127.13520240783691,
"meanCompletionTokens": 2,
"meanReasoningTokens": null
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"7": 24,
"42": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.018234106192829464,
"medianLatencyMs": 120.41724300384521,
"meanCompletionTokens": 2.04,
"meanReasoningTokens": null
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": -0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"cat": 21,
"dog": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6343095546405662,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"sea": 15,
"mountain": 10
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"tea": 15,
"coffee": 10
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9709505944546686,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"wednesday": 9,
"thursday": 14,
"tuesday": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.290564432903234,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"wednesday": 16,
"monday": 2,
"thursday": 1,
"friday": 1
},
"validCount": 20,
"invalidCount": 5,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.0750849518197798,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink