Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
EvalHarness/evalharness/fingerprint/references/tiangong_taie_fusion_reference.json
ruoxi_sun 09b2add673 fingerprint: integrate fp_fusion model fingerprint benchmark 
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1,
2026-09-07 pruning final): probe battery -> concurrent collection ->
five scoring views (verify/attribution/variant/adversarial/robustness),
bundled family aliases + 27 reference fingerprints (12 fp_fusion schema).

- CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single
  source of arg definitions) + 'fingerprint list' for bundled references
- imports rewritten package-relative; direct 'python3 run_fp_fusion.py'
  execution kept working via package bootstrap
- offline analysis/collection scripts made path-independent (previously
  pinned to a /opt/evalscope path absent on this host)
- shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars
- --reference accepts short names resolved against bundled references/
- pyproject: +httpx dependency, package-data references/*.json
- tests/test_fingerprint.py: 10 offline tests (battery definitions,
  assembly counts, normalization, signals, verdict ladder, CLI wiring)
- README: fingerprint section + architecture entry

Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs
vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
2026-09-11 03:52:21 +00:00

494 lines
12 KiB
JSON
Raw Blame History

{
"formatVersion": 1,
"protocol": "one-token/v1",
"model": "TianGong/Taie",
"collectedAt": "2026-09-02T05:51:59.665Z",
"samplesPerCell": 25,
"postReasoning": false,
"meta": {
"fusion": true,
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
"sourceDetector": "tiangong_taie_reference.json",
"sourceExtra": "/tmp/tg_extra_cells.json"
},
"cells": {
"random-number-1-100:en": {
"cellId": "random-number-1-100:en",
"counts": {
"17": 3,
"40": 3,
"42": 1,
"47": 3,
"57": 3,
"63": 1,
"70": 9,
"73": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.6619011889093374,
"normalizedEntropy": 0.4006560516776621,
"medianLatencyMs": 2153.2801619999955,
"meanCompletionTokens": 48.32,
"meanReasoningTokens": 36.48
},
"random-number-1-100:zh": {
"cellId": "random-number-1-100:zh",
"counts": {
"37": 4,
"42": 2,
"47": 7,
"57": 3,
"73": 9
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.1264283109928246,
"normalizedEntropy": 0.32005935261896845,
"medianLatencyMs": 1731.1657069999492,
"meanCompletionTokens": 32.28,
"meanReasoningTokens": 20.56
},
"random-color:en": {
"cellId": "random-color:en",
"counts": {
"turquoise": 17,
"teal": 8
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.9043814577244937,
"normalizedEntropy": 0.18430846176474383,
"medianLatencyMs": 1564.0879259999492,
"meanCompletionTokens": 20.6,
"meanReasoningTokens": 8.6
},
"random-animal:en": {
"cellId": "random-animal:en",
"counts": {
"axolotl": 9,
"pangolin": 6,
"capybara": 9,
"ocelot": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.7411191885631825,
"normalizedEntropy": 0.3084981491409475,
"medianLatencyMs": 1544.2841269999626,
"meanCompletionTokens": 21.4,
"meanReasoningTokens": 8.4
},
"random-number-1-10:en": {
"cellId": "random-number-1-10:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 1620.1512079999957,
"meanCompletionTokens": 27.76,
"meanReasoningTokens": 16.76
},
"random-letter:en": {
"cellId": "random-letter:en",
"counts": {
"q": 21,
"k": 4
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.6343095546405662,
"normalizedEntropy": 0.13494685448097182,
"medianLatencyMs": 1626.425771000002,
"meanCompletionTokens": 25.84,
"meanReasoningTokens": 14.84
},
"random-color:zh": {
"cellId": "random-color:zh",
"counts": {
"蓝": 17,
"靛蓝": 5,
"靛青": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.2098003386604828,
"normalizedEntropy": 0.2465513169874234,
"medianLatencyMs": 1503.029309000005,
"meanCompletionTokens": 13.32,
"meanReasoningTokens": 4.36
},
"coin-flip:en": {
"cellId": "coin-flip:en",
"counts": {
"heads": 22,
"tails": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.5293608652873644,
"normalizedEntropy": 0.5293608652873644,
"medianLatencyMs": 1512.3649179999993,
"meanCompletionTokens": 20.08,
"meanReasoningTokens": 8.96
},
"favorite-number:en": {
"cellId": "favorite-number:en",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 1452.4833170000347,
"meanCompletionTokens": 16.32,
"meanReasoningTokens": 6.76
},
"random-city:en": {
"cellId": "random-city:en",
"counts": {
"nairobi": 3,
"kyoto": 6,
"lisbon": 12,
"tokyo": 2,
"oslo": 1,
"marrakesh": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.0324876891689536,
"normalizedEntropy": 0.3601239331454476,
"medianLatencyMs": 1751.7330060000022,
"meanCompletionTokens": 20.88,
"meanReasoningTokens": 8.88
},
"random-number-1-10:zh": {
"cellId": "random-number-1-10:zh",
"counts": {
"6": 1,
"7": 24
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.07293721662889585,
"medianLatencyMs": 1697.2555729999876,
"meanCompletionTokens": 28.88,
"meanReasoningTokens": 17.88
},
"coin-flip:zh": {
"cellId": "coin-flip:zh",
"counts": {
"heads": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 1597.885345000017,
"meanCompletionTokens": 25.08,
"meanReasoningTokens": 13.08
},
"random-letter:zh": {
"cellId": "random-letter:zh",
"counts": {
"k": 16,
"q": 7,
"m": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.2177968115985955,
"normalizedEntropy": 0.2590814656974697,
"medianLatencyMs": 1573.4304589999956,
"meanCompletionTokens": 21.4,
"meanReasoningTokens": 10.4
},
"random-animal:zh": {
"cellId": "random-animal:zh",
"counts": {
"海豚": 18,
"水獭": 2,
"老虎": 3,
"熊猫": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.291314688649721,
"normalizedEntropy": 0.22880006953211615,
"medianLatencyMs": 1534.1851190000016,
"meanCompletionTokens": 17,
"meanReasoningTokens": 6.6
},
"random-city:zh": {
"cellId": "random-city:zh",
"counts": {
"巴黎": 6,
"苏州": 5,
"成都": 5,
"里斯本": 2,
"青岛": 2,
"北京": 3,
"南京": 1,
"杭州": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 2.744498451560163,
"normalizedEntropy": 0.48628072000355316,
"medianLatencyMs": 1592.624628999998,
"meanCompletionTokens": 15.32,
"meanReasoningTokens": 6.52
},
"favorite-number:zh": {
"cellId": "favorite-number:zh",
"counts": {
"7": 25
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0,
"normalizedEntropy": 0,
"medianLatencyMs": 1624.8507439999958,
"meanCompletionTokens": 19.48,
"meanReasoningTokens": 10.16
},
"binary-season:en": {
"cellId": "binary-season:en",
"counts": {
"summer": 20,
"winter": 5
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.7219280948873623,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-season:zh": {
"cellId": "binary-season:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:en": {
"cellId": "binary-pet:en",
"counts": {
"cat": 18,
"dog": 7
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.8554508105601306,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-pet:zh": {
"cellId": "binary-pet:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:en": {
"cellId": "binary-sea-mountain:en",
"counts": {
"mountain": 24,
"sea": 1
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.24229218908241482,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-sea-mountain:zh": {
"cellId": "binary-sea-mountain:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:en": {
"cellId": "binary-tea-coffee:en",
"counts": {
"coffee": 22,
"tea": 3
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.5293608652873644,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"binary-tea-coffee:zh": {
"cellId": "binary-tea-coffee:zh",
"counts": {},
"validCount": 0,
"invalidCount": 25,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.0,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:en": {
"cellId": "day-of-week:en",
"counts": {
"wednesday": 11,
"thursday": 12,
"tuesday": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 1.3209242772281589,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
},
"day-of-week:zh": {
"cellId": "day-of-week:zh",
"counts": {
"wednesday": 23,
"thursday": 2
},
"validCount": 25,
"invalidCount": 0,
"refusalCount": 0,
"emptyCount": 0,
"errorCount": 0,
"totalCount": 25,
"entropyBits": 0.4021791902022728,
"normalizedEntropy": 0.0,
"medianLatencyMs": null,
"meanCompletionTokens": null,
"meanReasoningTokens": null
}
}
}
Reference in New Issue View Git Blame Copy Permalink