370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
348 lines
8.6 KiB
JSON
348 lines
8.6 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "MoonshotAi/Kimi-K2.7-Code",
|
|
"collectedAt": "2026-09-04T09:56:49.375Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"37": 2,
|
|
"42": 8,
|
|
"47": 10,
|
|
"73": 5
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.810699332842307,
|
|
"normalizedEntropy": 0.27253740615714667,
|
|
"medianLatencyMs": 1929.3976545333862,
|
|
"meanCompletionTokens": 49.88,
|
|
"meanReasoningTokens": 46.88
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 2,
|
|
"42": 4,
|
|
"47": 1,
|
|
"73": 18
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2415101887362598,
|
|
"normalizedEntropy": 0.1868659033660324,
|
|
"medianLatencyMs": 4234.577438354492,
|
|
"meanCompletionTokens": 56.88,
|
|
"meanReasoningTokens": 53.88
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"magenta": 8,
|
|
"azure": 5,
|
|
"vermilion": 1,
|
|
"indigo": 4,
|
|
"cerulean": 1,
|
|
"cyan": 2,
|
|
"violet": 1,
|
|
"teal": 2,
|
|
"crimson": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7394705707972515,
|
|
"normalizedEntropy": 0.5582905339786818,
|
|
"medianLatencyMs": 1602.4216299057007,
|
|
"meanCompletionTokens": 44.2,
|
|
"meanReasoningTokens": 40.24
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"okapi": 1,
|
|
"octopus": 7,
|
|
"platypus": 4,
|
|
"elephant": 5,
|
|
"otter": 2,
|
|
"penguin": 4,
|
|
"pangolin": 1,
|
|
"tiger": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.673411192621123,
|
|
"normalizedEntropy": 0.47368520790176843,
|
|
"medianLatencyMs": 2826.1588563919067,
|
|
"meanCompletionTokens": 42.04,
|
|
"meanReasoningTokens": 37.76
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"6": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 1835.6529273986816,
|
|
"meanCompletionTokens": 55.8,
|
|
"meanReasoningTokens": 52.8
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"q": 18,
|
|
"m": 6,
|
|
"k": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.0211191885631823,
|
|
"normalizedEntropy": 0.21723907757442953,
|
|
"medianLatencyMs": 2027.0736074447632,
|
|
"meanCompletionTokens": 43,
|
|
"meanReasoningTokens": 40
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"靛蓝": 2,
|
|
"蓝": 14,
|
|
"紫": 6,
|
|
"橙": 2,
|
|
"青": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7313464332493889,
|
|
"normalizedEntropy": 0.3528398278940391,
|
|
"medianLatencyMs": 1975.8200035095215,
|
|
"meanCompletionTokens": 69.88,
|
|
"meanReasoningTokens": 66.72
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1818.241452217102,
|
|
"meanCompletionTokens": 51.68,
|
|
"meanReasoningTokens": 47.92
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 2659.0778017044067,
|
|
"meanCompletionTokens": 63.36,
|
|
"meanReasoningTokens": 60.36
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"kyoto": 4,
|
|
"osaka": 2,
|
|
"lisbon": 8,
|
|
"paris": 2,
|
|
"tokyo": 2,
|
|
"budapest": 2,
|
|
"tbilisi": 1,
|
|
"timbuktu": 2,
|
|
"mumbai": 1,
|
|
"baku": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.963856189774724,
|
|
"normalizedEntropy": 0.5251473620367048,
|
|
"medianLatencyMs": 2277.6057929992676,
|
|
"meanCompletionTokens": 46.44,
|
|
"meanReasoningTokens": 41.92
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"3": 1,
|
|
"7": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.07293721662889585,
|
|
"medianLatencyMs": 2840.0440530776978,
|
|
"meanCompletionTokens": 70.4,
|
|
"meanReasoningTokens": 67.4
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 19,
|
|
"tails": 6
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7950402793845223,
|
|
"normalizedEntropy": 0.7950402793845223,
|
|
"medianLatencyMs": 2722.0782718658447,
|
|
"meanCompletionTokens": 58.96,
|
|
"meanReasoningTokens": 55.96
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"x": 3,
|
|
"k": 6,
|
|
"m": 9,
|
|
"q": 6,
|
|
"l": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0717056888227985,
|
|
"normalizedEntropy": 0.44074720942110224,
|
|
"medianLatencyMs": 1683.4700717926025,
|
|
"meanCompletionTokens": 47.4,
|
|
"meanReasoningTokens": 44.4
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"长颈鹿": 5,
|
|
"海豚": 1,
|
|
"企鹅": 5,
|
|
"老虎": 4,
|
|
"熊猫": 3,
|
|
"猫": 5,
|
|
"树懒": 1,
|
|
"大象": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.7405038327557683,
|
|
"normalizedEntropy": 0.48557293818380515,
|
|
"medianLatencyMs": 2110.104063987732,
|
|
"meanCompletionTokens": 69.04,
|
|
"meanReasoningTokens": 66
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"成都": 5,
|
|
"杭州": 7,
|
|
"北京": 3,
|
|
"京都": 2,
|
|
"西安": 1,
|
|
"上海": 1,
|
|
"桂林": 1,
|
|
"墨尔本": 1,
|
|
"喀什": 1,
|
|
"雷克雅未克": 1,
|
|
"拉萨": 1,
|
|
"苏州": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 3.123215692534583,
|
|
"normalizedEntropy": 0.5533832875105995,
|
|
"medianLatencyMs": 3302.0929412841797,
|
|
"meanCompletionTokens": 85.04,
|
|
"meanReasoningTokens": 81.88
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 24,
|
|
"42": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.018234106192829464,
|
|
"medianLatencyMs": 3258.5312995910645,
|
|
"meanCompletionTokens": 86.6,
|
|
"meanReasoningTokens": 83.6
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
}
|