Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/glm52_vectron_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

522 lines
13 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "ZhipuAi/GLM-5.2",
"collectedAt": "2026-09-02T02:19:58.189Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
"sourceDetector": "glm52_vectron_reference.json",
"sourceExtra": "/tmp/g52_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"42": 15,
"47": 1,
"57": 1,
"73": 8
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.3397218324096136,
"normalizedEntropy": 0.20164822870060348,
"medianLatencyMs": 2865.177502000006,
"meanCompletionTokens": 148.24,
"meanReasoningTokens": 145.32
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"42": 20,
"57": 1,
"58": 1,
"73": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.996118213778296,
"normalizedEntropy": 0.14993073078724659,
"medianLatencyMs": 3416.6582340000023,
"meanCompletionTokens": 217.12,
"meanReasoningTokens": 214.28
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"teal": 5,
"blue": 5,
"magenta": 3,
"purple": 7,
"cerulean": 1,
"azure": 1,
"crimson": 1,
"green": 1,
"violet": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.738830073557111,
"normalizedEntropy": 0.558160003813466,
"medianLatencyMs": 2797.5234410000267,
"meanCompletionTokens": 153.6,
"meanReasoningTokens": 150.36
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"kangaroo": 1,
"zebra": 3,
"elephant": 5,
"jaguar": 1,
"hippopotamus": 1,
"giraffe": 3,
"capybara": 4,
"penguin": 2,
"platypus": 3,
"fox": 1,
"tiger": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.2088840705376356,
"normalizedEntropy": 0.5685623379899973,
"medianLatencyMs": 3114.7327939999523,
"meanCompletionTokens": 168.24,
"meanReasoningTokens": 163.8
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": null,
"meanCompletionTokens": 207.88,
"meanReasoningTokens": 204.92
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"r": 3,
"k": 10,
"q": 7,
"m": 3,
"g": 1,
"j": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.148634573470573,
"normalizedEntropy": 0.4571135260341782,
"medianLatencyMs": 2339.990761999972,
"meanCompletionTokens": 165.84,
"meanReasoningTokens": 162.92
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"蓝": 15,
"青": 2,
"紫": 3,
"绿": 1,
"红": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.709526332323075,
"normalizedEntropy": 0.34839299939824137,
"medianLatencyMs": 4651.723928000021,
"meanCompletionTokens": 284.68,
"meanReasoningTokens": 281.68
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 2993.0387099999934,
"meanCompletionTokens": 187.24,
"meanReasoningTokens": 183.96
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 12,
"42": 13
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9988455359952018,
"normalizedEntropy": 0.07516980073746855,
"medianLatencyMs": 4419.354362999991,
"meanCompletionTokens": 284.12,
"meanReasoningTokens": 281.16
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"paris": 1,
"austin": 1,
"barcelona": 2,
"seattle": 2,
"tokyo": 8,
"stockholm": 1,
"oslo": 4,
"nairobi": 1,
"madrid": 1,
"berlin": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.883856189774724,
"normalizedEntropy": 0.5109726564258601,
"medianLatencyMs": 2799.2111550000263,
"meanCompletionTokens": 160.16,
"meanReasoningTokens": 156.52
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 3109.8583069999004,
"meanCompletionTokens": 208.48,
"meanReasoningTokens": 205.72
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 3598.706825000001,
"meanCompletionTokens": 215.72,
"meanReasoningTokens": 212.8
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"m": 8,
"j": 1,
"q": 7,
"k": 8,
"r": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.9377968115985953,
"normalizedEntropy": 0.41225862425589116,
"medianLatencyMs": null,
"meanCompletionTokens": 235.4,
"meanReasoningTokens": 232.52
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"猫": 20,
"狐狸": 2,
"老虎": 1,
"狼": 1,
"长颈鹿": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.106313713864835,
"normalizedEntropy": 0.196020890090928,
"medianLatencyMs": 4062.5094319999916,
"meanCompletionTokens": 249.48,
"meanReasoningTokens": 246.56
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"伦敦": 1,
"东京": 6,
"北京": 8,
"巴黎": 4,
"柏林": 2,
"成都": 1,
"厦门": 1,
"杭州": 1,
"深圳": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.663465189601647,
"normalizedEntropy": 0.47192293709169786,
"medianLatencyMs": 4759.383081000007,
"meanCompletionTokens": 261.96,
"meanReasoningTokens": 259
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"0": 1,
"1": 1,
"7": 14,
"8": 7,
"42": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.6456780552463373,
"normalizedEntropy": 0.12384826986050242,
"medianLatencyMs": 6356.738842000021,
"meanCompletionTokens": 367.8,
"meanReasoningTokens": 364.96
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 21,
"winter": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6343095546405662,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"cat": 3,
"dog": 20
},
"validCount": 23,
"invalidCount": 2,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.624609718596318,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"mountain": 14,
"sea": 11
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9895875212220556,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"tea": 7,
"coffee": 18
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.8554508105601306,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"wednesday": 9,
"tuesday": 1,
"thursday": 15
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.1585488318903812,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"thursday": 3,
"wednesday": 18,
"monday": 2,
"friday": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.291314688649721,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink