Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/glm_51_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

516 lines
13 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "ZhipuAi/GLM-5.1",
"collectedAt": "2026-09-04T10:18:07.045Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
"sourceDetector": "glm_51_tmp_reference.json",
"sourceExtra": "/tmp/bfd/glm_51_tmp_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"42": 14,
"47": 1,
"67": 1,
"73": 5,
"77": 1,
"83": 1,
"87": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.967351814444994,
"normalizedEntropy": 0.29611595408595104,
"medianLatencyMs": 3679.5200386047363,
"meanCompletionTokens": 282.96,
"meanReasoningTokens": 279.92
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"42": 22,
"57": 1,
"73": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6395563653739031,
"normalizedEntropy": 0.09626282494768883,
"medianLatencyMs": 3838.964512825012,
"meanCompletionTokens": 183.44,
"meanReasoningTokens": 180.44
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"magenta": 5,
"teal": 11,
"green": 3,
"cerulean": 1,
"violet": 1,
"blue": 2,
"turquoise": 1,
"periwinkle": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.3871251585103024,
"normalizedEntropy": 0.4864842840895391,
"medianLatencyMs": 3823.377359390259,
"meanCompletionTokens": 201.12,
"meanReasoningTokens": 197.24
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"axolotl": 2,
"sloth": 2,
"capybara": 8,
"pangolin": 3,
"octopus": 1,
"fox": 1,
"platypus": 2,
"dolphin": 1,
"tiger": 1,
"giraffe": 1,
"quokka": 2,
"otter": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 3.173660689688185,
"normalizedEntropy": 0.5623213248130021,
"medianLatencyMs": 3547.2432861328125,
"meanCompletionTokens": 183.96,
"meanReasoningTokens": 179.08
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 3413.499443054199,
"meanCompletionTokens": 215.4,
"meanReasoningTokens": 212.36
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"m": 6,
"z": 1,
"k": 11,
"q": 6,
"f": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.8809242772281591,
"normalizedEntropy": 0.4001592170130029,
"medianLatencyMs": 3375.534640312195,
"meanCompletionTokens": 170.28,
"meanReasoningTokens": 167.24
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"紫": 8,
"蓝": 11,
"红": 4,
"绛红": 1,
"靛蓝": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.841706277574314,
"normalizedEntropy": 0.3753306175651382,
"medianLatencyMs": 4193.752639770508,
"meanCompletionTokens": 263.16,
"meanReasoningTokens": 258.4
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"tails": 1,
"heads": 24
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.24229218908241482,
"medianLatencyMs": 3576.8441257476807,
"meanCompletionTokens": 178.32,
"meanReasoningTokens": 174.92
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 16,
"42": 9
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9426831892554922,
"normalizedEntropy": 0.07094320887592885,
"medianLatencyMs": 6315.492043495178,
"meanCompletionTokens": 359.44,
"meanReasoningTokens": 356.44
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"wellington": 1,
"nairobi": 2,
"tokyo": 5,
"oslo": 10,
"seattle": 1,
"denver": 2,
"kyoto": 2,
"berlin": 1,
"chicago": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.610699332842307,
"normalizedEntropy": 0.4625736810183524,
"medianLatencyMs": 3581.210454940796,
"meanCompletionTokens": 170.92,
"meanReasoningTokens": 167.12
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 3535.8241214752197,
"meanCompletionTokens": 169.8,
"meanReasoningTokens": 166.68
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 3879.5468101501465,
"meanCompletionTokens": 198.4,
"meanReasoningTokens": 195.28
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"k": 8,
"q": 11,
"m": 4,
"r": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.761706277574314,
"normalizedEntropy": 0.3747960580741211,
"medianLatencyMs": 4251.800204277039,
"meanCompletionTokens": 220.44,
"meanReasoningTokens": 217.36
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"熊猫": 1,
"猫": 22,
"斑马": 1,
"水豚": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7195563653739032,
"normalizedEntropy": 0.12749374561980548,
"medianLatencyMs": 4210.798274040222,
"meanCompletionTokens": 231.76,
"meanReasoningTokens": 228.4
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"北京": 9,
"巴黎": 6,
"伦敦": 3,
"成都": 2,
"上海": 2,
"武汉": 1,
"东京": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.452096688995876,
"normalizedEntropy": 0.4344718586980424,
"medianLatencyMs": 4630.0859479904175,
"meanCompletionTokens": 247.96,
"meanReasoningTokens": 244.96
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"0": 2,
"1": 2,
"7": 15,
"8": 4,
"42": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.7397218324096138,
"normalizedEntropy": 0.13092569248012592,
"medianLatencyMs": 6921.807636260986,
"meanCompletionTokens": 322.2,
"meanReasoningTokens": 319.12
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": -0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"dog": 12,
"cat": 12
},
"validCount": 24,
"invalidCount": 1,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.0165379414914257,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"mountain": 17,
"sea": 8
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9043814577244937,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"tea": 6,
"coffee": 19
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7950402793845223,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"thursday": 16,
"wednesday": 9
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9426831892554922,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"thursday": 3,
"wednesday": 18,
"friday": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.131314688649721,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink