370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
319 lines
7.9 KiB
JSON
319 lines
7.9 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "Qwen3.8-27B",
|
|
"collectedAt": "2026-09-03T10:20:36.636Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"42": 23,
|
|
"47": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4021791902022728,
|
|
"normalizedEntropy": 0.06053399994136682,
|
|
"medianLatencyMs": 182.66270351409912,
|
|
"meanCompletionTokens": 3,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"7": 2,
|
|
"42": 20,
|
|
"47": 2,
|
|
"73": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.0263137138648348,
|
|
"normalizedEntropy": 0.15447560641730784,
|
|
"medianLatencyMs": 178.68310260772705,
|
|
"meanCompletionTokens": 2.92,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"blue": 14,
|
|
"purple": 1,
|
|
"cobalt": 2,
|
|
"teal": 6,
|
|
"turquoise": 1,
|
|
"indigo": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.811346433249389,
|
|
"normalizedEntropy": 0.3691434316612796,
|
|
"medianLatencyMs": 143.06485271453857,
|
|
"meanCompletionTokens": 2.48,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"kangaroo": 2,
|
|
"platypus": 5,
|
|
"koala": 1,
|
|
"ostrich": 7,
|
|
"falcon": 1,
|
|
"otter": 4,
|
|
"fox": 4,
|
|
"elephant": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.673411192621123,
|
|
"normalizedEntropy": 0.47368520790176843,
|
|
"medianLatencyMs": 190.27048015594482,
|
|
"meanCompletionTokens": 3.6,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 123.02077293395996,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"q": 16,
|
|
"k": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.20055212826520413,
|
|
"medianLatencyMs": 122.49901103973389,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 22,
|
|
"红": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.5293608652873644,
|
|
"normalizedEntropy": 0.10788112246910952,
|
|
"medianLatencyMs": 127.85008716583252,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 180.7693395614624,
|
|
"meanCompletionTokens": 3,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 126.00289821624756,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"kyoto": 11,
|
|
"paris": 2,
|
|
"lisbon": 2,
|
|
"osaka": 4,
|
|
"tokyo": 3,
|
|
"oslo": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.2613152774012364,
|
|
"normalizedEntropy": 0.40066847938084993,
|
|
"medianLatencyMs": 181.48858451843262,
|
|
"meanCompletionTokens": 3,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 127.78436660766602,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 22
|
|
},
|
|
"validCount": 22,
|
|
"invalidCount": 3,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 99.95673847198486,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"q": 13,
|
|
"k": 11,
|
|
"m": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.1974776241409462,
|
|
"normalizedEntropy": 0.2547586387544438,
|
|
"medianLatencyMs": 127.07363319396973,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"猫": 14,
|
|
"狐狸": 2,
|
|
"狐": 5,
|
|
"熊猫": 1,
|
|
"鲸": 1,
|
|
"海豚": 1,
|
|
"鲸鱼": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.967351814444994,
|
|
"normalizedEntropy": 0.34858291003398534,
|
|
"medianLatencyMs": 139.31216621398926,
|
|
"meanCompletionTokens": 2.04,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"巴黎": 11,
|
|
"东京": 13,
|
|
"成都": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.1974776241409462,
|
|
"normalizedEntropy": 0.21217365997214463,
|
|
"medianLatencyMs": 127.13520240783691,
|
|
"meanCompletionTokens": 2,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 24,
|
|
"42": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.018234106192829464,
|
|
"medianLatencyMs": 120.41724300384521,
|
|
"meanCompletionTokens": 2.04,
|
|
"meanReasoningTokens": null
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
} |