370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
334 lines
8.3 KiB
JSON
334 lines
8.3 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "MoonshotAi/Kimi-K3",
|
|
"collectedAt": "2026-09-01T08:25:58.034Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"37": 6,
|
|
"42": 9,
|
|
"47": 7,
|
|
"73": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.9060373108197468,
|
|
"normalizedEntropy": 0.2868872017057274,
|
|
"medianLatencyMs": 4201.567929000012,
|
|
"meanCompletionTokens": 54.92,
|
|
"meanReasoningTokens": 40.52
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 7,
|
|
"42": 4,
|
|
"47": 9,
|
|
"57": 4,
|
|
"73": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0766238110793633,
|
|
"normalizedEntropy": 0.31256302842247047,
|
|
"medianLatencyMs": 4935.542820999981,
|
|
"meanCompletionTokens": 47.76,
|
|
"meanReasoningTokens": 33.76
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"coral": 1,
|
|
"crimson": 3,
|
|
"blue": 7,
|
|
"chartreuse": 2,
|
|
"cerulean": 2,
|
|
"azure": 8,
|
|
"teal": 1,
|
|
"indigo": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.5476013115120564,
|
|
"normalizedEntropy": 0.5191885292474349,
|
|
"medianLatencyMs": 4126.816009999951,
|
|
"meanCompletionTokens": 31.76,
|
|
"meanReasoningTokens": 16.44
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"otter": 17,
|
|
"elephant": 2,
|
|
"penguin": 1,
|
|
"capybara": 1,
|
|
"pangolin": 1,
|
|
"octopus": 2,
|
|
"platypus": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.704381457724494,
|
|
"normalizedEntropy": 0.30198881764783675,
|
|
"medianLatencyMs": 4583.067276999936,
|
|
"meanCompletionTokens": 26,
|
|
"meanReasoningTokens": 10.88
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 4282.95300400001,
|
|
"meanCompletionTokens": 40.88,
|
|
"meanReasoningTokens": 25.92
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"m": 4,
|
|
"q": 19,
|
|
"k": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.0154312795575997,
|
|
"normalizedEntropy": 0.2160289973805212,
|
|
"medianLatencyMs": 4654.510852000036,
|
|
"meanCompletionTokens": 38,
|
|
"meanReasoningTokens": 23.72
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 23,
|
|
"蔚蓝": 1,
|
|
"紫": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.48217919020227284,
|
|
"normalizedEntropy": 0.09826573077333434,
|
|
"medianLatencyMs": 3668.3515180000104,
|
|
"meanCompletionTokens": 40.52,
|
|
"meanReasoningTokens": 28.32
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 23,
|
|
"tails": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4021791902022728,
|
|
"normalizedEntropy": 0.4021791902022728,
|
|
"medianLatencyMs": 5788.457129999995,
|
|
"meanCompletionTokens": 57.12,
|
|
"meanReasoningTokens": 42.28
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 21,
|
|
"42": 2
|
|
},
|
|
"validCount": 23,
|
|
"invalidCount": 2,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4262286569981449,
|
|
"normalizedEntropy": 0.032076554442651374,
|
|
"medianLatencyMs": 4735.351004000055,
|
|
"meanCompletionTokens": 69.8,
|
|
"meanReasoningTokens": 49.6
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"timbuktu": 2,
|
|
"tokyo": 5,
|
|
"lisbon": 11,
|
|
"osaka": 1,
|
|
"reykjavik": 2,
|
|
"kyoto": 3,
|
|
"tucson": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.3071251585103023,
|
|
"normalizedEntropy": 0.40878524911570996,
|
|
"medianLatencyMs": 4120.0932230000035,
|
|
"meanCompletionTokens": 34.16,
|
|
"meanReasoningTokens": 18.08
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 5510.148357999977,
|
|
"meanCompletionTokens": 52.8,
|
|
"meanReasoningTokens": 37.36
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 18,
|
|
"tails": 7
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8554508105601306,
|
|
"normalizedEntropy": 0.8554508105601306,
|
|
"medianLatencyMs": 3399.379054000019,
|
|
"meanCompletionTokens": 62.36,
|
|
"meanReasoningTokens": 47.28
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 4,
|
|
"q": 16,
|
|
"m": 5
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.2994705707972523,
|
|
"normalizedEntropy": 0.2764572356458516,
|
|
"medianLatencyMs": 4990.088311000029,
|
|
"meanCompletionTokens": 49.76,
|
|
"meanReasoningTokens": 34.84
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"水獭": 2,
|
|
"水豚": 2,
|
|
"熊猫": 8,
|
|
"猫": 11,
|
|
"海豚": 1,
|
|
"狐狸": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.0017062775743137,
|
|
"normalizedEntropy": 0.35466996504994436,
|
|
"medianLatencyMs": 5375.511597000004,
|
|
"meanCompletionTokens": 51.52,
|
|
"meanReasoningTokens": 34.84
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"杭州": 1,
|
|
"西安": 4,
|
|
"昆明": 4,
|
|
"北京": 3,
|
|
"成都": 5,
|
|
"巴黎": 5,
|
|
"雷克雅未克": 1,
|
|
"青岛": 1,
|
|
"维也纳": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.8848894517332404,
|
|
"normalizedEntropy": 0.5111557337268707,
|
|
"medianLatencyMs": 4517.09676100011,
|
|
"meanCompletionTokens": 48.16,
|
|
"meanReasoningTokens": 34.12
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 22
|
|
},
|
|
"validCount": 22,
|
|
"invalidCount": 3,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 4412.280236000079,
|
|
"meanCompletionTokens": 64.36,
|
|
"meanReasoningTokens": 41.2
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
}
|