370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
323 lines
8.1 KiB
JSON
323 lines
8.1 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "TianGong/Taie",
|
|
"collectedAt": "2026-09-02T05:51:59.665Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"17": 3,
|
|
"40": 3,
|
|
"42": 1,
|
|
"47": 3,
|
|
"57": 3,
|
|
"63": 1,
|
|
"70": 9,
|
|
"73": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.6619011889093374,
|
|
"normalizedEntropy": 0.4006560516776621,
|
|
"medianLatencyMs": 2153.2801619999955,
|
|
"meanCompletionTokens": 48.32,
|
|
"meanReasoningTokens": 36.48
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 4,
|
|
"42": 2,
|
|
"47": 7,
|
|
"57": 3,
|
|
"73": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.1264283109928246,
|
|
"normalizedEntropy": 0.32005935261896845,
|
|
"medianLatencyMs": 1731.1657069999492,
|
|
"meanCompletionTokens": 32.28,
|
|
"meanReasoningTokens": 20.56
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"turquoise": 17,
|
|
"teal": 8
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9043814577244937,
|
|
"normalizedEntropy": 0.18430846176474383,
|
|
"medianLatencyMs": 1564.0879259999492,
|
|
"meanCompletionTokens": 20.6,
|
|
"meanReasoningTokens": 8.6
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"axolotl": 9,
|
|
"pangolin": 6,
|
|
"capybara": 9,
|
|
"ocelot": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7411191885631825,
|
|
"normalizedEntropy": 0.3084981491409475,
|
|
"medianLatencyMs": 1544.2841269999626,
|
|
"meanCompletionTokens": 21.4,
|
|
"meanReasoningTokens": 8.4
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1620.1512079999957,
|
|
"meanCompletionTokens": 27.76,
|
|
"meanReasoningTokens": 16.76
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"q": 21,
|
|
"k": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6343095546405662,
|
|
"normalizedEntropy": 0.13494685448097182,
|
|
"medianLatencyMs": 1626.425771000002,
|
|
"meanCompletionTokens": 25.84,
|
|
"meanReasoningTokens": 14.84
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 17,
|
|
"靛蓝": 5,
|
|
"靛青": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2098003386604828,
|
|
"normalizedEntropy": 0.2465513169874234,
|
|
"medianLatencyMs": 1503.029309000005,
|
|
"meanCompletionTokens": 13.32,
|
|
"meanReasoningTokens": 4.36
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 22,
|
|
"tails": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.5293608652873644,
|
|
"normalizedEntropy": 0.5293608652873644,
|
|
"medianLatencyMs": 1512.3649179999993,
|
|
"meanCompletionTokens": 20.08,
|
|
"meanReasoningTokens": 8.96
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1452.4833170000347,
|
|
"meanCompletionTokens": 16.32,
|
|
"meanReasoningTokens": 6.76
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"nairobi": 3,
|
|
"kyoto": 6,
|
|
"lisbon": 12,
|
|
"tokyo": 2,
|
|
"oslo": 1,
|
|
"marrakesh": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0324876891689536,
|
|
"normalizedEntropy": 0.3601239331454476,
|
|
"medianLatencyMs": 1751.7330060000022,
|
|
"meanCompletionTokens": 20.88,
|
|
"meanReasoningTokens": 8.88
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"6": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 1697.2555729999876,
|
|
"meanCompletionTokens": 28.88,
|
|
"meanReasoningTokens": 17.88
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1597.885345000017,
|
|
"meanCompletionTokens": 25.08,
|
|
"meanReasoningTokens": 13.08
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 16,
|
|
"q": 7,
|
|
"m": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2177968115985955,
|
|
"normalizedEntropy": 0.2590814656974697,
|
|
"medianLatencyMs": 1573.4304589999956,
|
|
"meanCompletionTokens": 21.4,
|
|
"meanReasoningTokens": 10.4
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"海豚": 18,
|
|
"水獭": 2,
|
|
"老虎": 3,
|
|
"熊猫": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.291314688649721,
|
|
"normalizedEntropy": 0.22880006953211615,
|
|
"medianLatencyMs": 1534.1851190000016,
|
|
"meanCompletionTokens": 17,
|
|
"meanReasoningTokens": 6.6
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"巴黎": 6,
|
|
"苏州": 5,
|
|
"成都": 5,
|
|
"里斯本": 2,
|
|
"青岛": 2,
|
|
"北京": 3,
|
|
"南京": 1,
|
|
"杭州": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.744498451560163,
|
|
"normalizedEntropy": 0.48628072000355316,
|
|
"medianLatencyMs": 1592.624628999998,
|
|
"meanCompletionTokens": 15.32,
|
|
"meanReasoningTokens": 6.52
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1624.8507439999958,
|
|
"meanCompletionTokens": 19.48,
|
|
"meanReasoningTokens": 10.16
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
}
|