370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
336 lines
8.2 KiB
JSON
336 lines
8.2 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "DeepSeek/DeepSeek-V4-Flash",
|
|
"collectedAt": "2026-09-01T06:53:23.825Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"5": 1,
|
|
"7": 2,
|
|
"12": 1,
|
|
"17": 1,
|
|
"23": 1,
|
|
"37": 1,
|
|
"42": 6,
|
|
"57": 1,
|
|
"70": 1,
|
|
"73": 9,
|
|
"80": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.8022921890824146,
|
|
"normalizedEntropy": 0.42178700276434383,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 243.48,
|
|
"meanReasoningTokens": 241.24
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"23": 1,
|
|
"37": 5,
|
|
"42": 14,
|
|
"47": 2,
|
|
"57": 1,
|
|
"73": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.887351814444994,
|
|
"normalizedEntropy": 0.28407475425939177,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 54.56,
|
|
"meanReasoningTokens": 52.56
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"blue": 22,
|
|
"cyan": 1,
|
|
"red": 1,
|
|
"magenta": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7195563653739032,
|
|
"normalizedEntropy": 0.14664202336564808,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 44.64,
|
|
"meanReasoningTokens": 42.6
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"giraffe": 5,
|
|
"elephant": 13,
|
|
"cat": 3,
|
|
"penguin": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7450464172773457,
|
|
"normalizedEntropy": 0.309193990527069,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 42.48,
|
|
"meanReasoningTokens": 39.32
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"3": 1,
|
|
"4": 2,
|
|
"5": 1,
|
|
"7": 21
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8743095546405661,
|
|
"normalizedEntropy": 0.263193401442427,
|
|
"medianLatencyMs": 1554.6371949999884,
|
|
"meanCompletionTokens": 73.36,
|
|
"meanReasoningTokens": 71.36
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"m": 12,
|
|
"g": 2,
|
|
"k": 7,
|
|
"q": 1,
|
|
"x": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.866819311165902,
|
|
"normalizedEntropy": 0.3971584411477535,
|
|
"medianLatencyMs": 1480.1575740000117,
|
|
"meanCompletionTokens": 56.04,
|
|
"meanReasoningTokens": 54.04
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 23,
|
|
"绿": 1,
|
|
"紫": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.48217919020227284,
|
|
"normalizedEntropy": 0.09826573077333434,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 37.72,
|
|
"meanReasoningTokens": 35.72
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 24,
|
|
"tails": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.24229218908241482,
|
|
"medianLatencyMs": 1675.2963849999942,
|
|
"meanCompletionTokens": 73.16,
|
|
"meanReasoningTokens": 71.16
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 116.52,
|
|
"meanReasoningTokens": 114.52
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"tokyo": 19,
|
|
"london": 3,
|
|
"cairo": 1,
|
|
"kyoto": 1,
|
|
"paris": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.225235779471061,
|
|
"normalizedEntropy": 0.2170919559734506,
|
|
"medianLatencyMs": 1439.2404779999924,
|
|
"meanCompletionTokens": 47.64,
|
|
"meanReasoningTokens": 45.56
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"5": 3,
|
|
"7": 21,
|
|
"8": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7641140545540274,
|
|
"normalizedEntropy": 0.23002125052918596,
|
|
"medianLatencyMs": 1451.3741049999371,
|
|
"meanCompletionTokens": 47.16,
|
|
"meanReasoningTokens": 45.16
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 1457.2705060000008,
|
|
"meanCompletionTokens": 50.92,
|
|
"meanReasoningTokens": 48.92
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 8,
|
|
"q": 1,
|
|
"a": 9,
|
|
"m": 6,
|
|
"g": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.9222921890824147,
|
|
"normalizedEntropy": 0.40896007700373915,
|
|
"medianLatencyMs": 1475.0125490000937,
|
|
"meanCompletionTokens": 33.8,
|
|
"meanReasoningTokens": 31.8
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"熊猫": 6,
|
|
"老虎": 2,
|
|
"猫": 11,
|
|
"大象": 4,
|
|
"狗": 1,
|
|
"袋鼠": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.1013152774012362,
|
|
"normalizedEntropy": 0.37231906815916066,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 35.96,
|
|
"meanReasoningTokens": 33.92
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"巴黎": 5,
|
|
"东京": 15,
|
|
"北京": 2,
|
|
"伦敦": 1,
|
|
"里斯本": 1,
|
|
"上海": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7553362134321413,
|
|
"normalizedEntropy": 0.31101717591819183,
|
|
"medianLatencyMs": 1345.1831369999563,
|
|
"meanCompletionTokens": 33.56,
|
|
"meanReasoningTokens": 31.52
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 37.88,
|
|
"meanReasoningTokens": 35.88
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
} |