370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
346 lines
8.6 KiB
JSON
346 lines
8.6 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "ZhipuAi/GLM-5.1",
|
|
"collectedAt": "2026-09-04T10:18:07.045Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"42": 14,
|
|
"47": 1,
|
|
"67": 1,
|
|
"73": 5,
|
|
"77": 1,
|
|
"83": 1,
|
|
"87": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.967351814444994,
|
|
"normalizedEntropy": 0.29611595408595104,
|
|
"medianLatencyMs": 3679.5200386047363,
|
|
"meanCompletionTokens": 282.96,
|
|
"meanReasoningTokens": 279.92
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"42": 22,
|
|
"57": 1,
|
|
"73": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6395563653739031,
|
|
"normalizedEntropy": 0.09626282494768883,
|
|
"medianLatencyMs": 3838.964512825012,
|
|
"meanCompletionTokens": 183.44,
|
|
"meanReasoningTokens": 180.44
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"magenta": 5,
|
|
"teal": 11,
|
|
"green": 3,
|
|
"cerulean": 1,
|
|
"violet": 1,
|
|
"blue": 2,
|
|
"turquoise": 1,
|
|
"periwinkle": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.3871251585103024,
|
|
"normalizedEntropy": 0.4864842840895391,
|
|
"medianLatencyMs": 3823.377359390259,
|
|
"meanCompletionTokens": 201.12,
|
|
"meanReasoningTokens": 197.24
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"axolotl": 2,
|
|
"sloth": 2,
|
|
"capybara": 8,
|
|
"pangolin": 3,
|
|
"octopus": 1,
|
|
"fox": 1,
|
|
"platypus": 2,
|
|
"dolphin": 1,
|
|
"tiger": 1,
|
|
"giraffe": 1,
|
|
"quokka": 2,
|
|
"otter": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 3.173660689688185,
|
|
"normalizedEntropy": 0.5623213248130021,
|
|
"medianLatencyMs": 3547.2432861328125,
|
|
"meanCompletionTokens": 183.96,
|
|
"meanReasoningTokens": 179.08
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 3413.499443054199,
|
|
"meanCompletionTokens": 215.4,
|
|
"meanReasoningTokens": 212.36
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"m": 6,
|
|
"z": 1,
|
|
"k": 11,
|
|
"q": 6,
|
|
"f": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.8809242772281591,
|
|
"normalizedEntropy": 0.4001592170130029,
|
|
"medianLatencyMs": 3375.534640312195,
|
|
"meanCompletionTokens": 170.28,
|
|
"meanReasoningTokens": 167.24
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"紫": 8,
|
|
"蓝": 11,
|
|
"红": 4,
|
|
"绛红": 1,
|
|
"靛蓝": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.841706277574314,
|
|
"normalizedEntropy": 0.3753306175651382,
|
|
"medianLatencyMs": 4193.752639770508,
|
|
"meanCompletionTokens": 263.16,
|
|
"meanReasoningTokens": 258.4
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"tails": 1,
|
|
"heads": 24
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.24229218908241482,
|
|
"medianLatencyMs": 3576.8441257476807,
|
|
"meanCompletionTokens": 178.32,
|
|
"meanReasoningTokens": 174.92
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 16,
|
|
"42": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.07094320887592885,
|
|
"medianLatencyMs": 6315.492043495178,
|
|
"meanCompletionTokens": 359.44,
|
|
"meanReasoningTokens": 356.44
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"wellington": 1,
|
|
"nairobi": 2,
|
|
"tokyo": 5,
|
|
"oslo": 10,
|
|
"seattle": 1,
|
|
"denver": 2,
|
|
"kyoto": 2,
|
|
"berlin": 1,
|
|
"chicago": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.610699332842307,
|
|
"normalizedEntropy": 0.4625736810183524,
|
|
"medianLatencyMs": 3581.210454940796,
|
|
"meanCompletionTokens": 170.92,
|
|
"meanReasoningTokens": 167.12
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 3535.8241214752197,
|
|
"meanCompletionTokens": 169.8,
|
|
"meanReasoningTokens": 166.68
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 3879.5468101501465,
|
|
"meanCompletionTokens": 198.4,
|
|
"meanReasoningTokens": 195.28
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 8,
|
|
"q": 11,
|
|
"m": 4,
|
|
"r": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.761706277574314,
|
|
"normalizedEntropy": 0.3747960580741211,
|
|
"medianLatencyMs": 4251.800204277039,
|
|
"meanCompletionTokens": 220.44,
|
|
"meanReasoningTokens": 217.36
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"熊猫": 1,
|
|
"猫": 22,
|
|
"斑马": 1,
|
|
"水豚": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7195563653739032,
|
|
"normalizedEntropy": 0.12749374561980548,
|
|
"medianLatencyMs": 4210.798274040222,
|
|
"meanCompletionTokens": 231.76,
|
|
"meanReasoningTokens": 228.4
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"北京": 9,
|
|
"巴黎": 6,
|
|
"伦敦": 3,
|
|
"成都": 2,
|
|
"上海": 2,
|
|
"武汉": 1,
|
|
"东京": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.452096688995876,
|
|
"normalizedEntropy": 0.4344718586980424,
|
|
"medianLatencyMs": 4630.0859479904175,
|
|
"meanCompletionTokens": 247.96,
|
|
"meanReasoningTokens": 244.96
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"0": 2,
|
|
"1": 2,
|
|
"7": 15,
|
|
"8": 4,
|
|
"42": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7397218324096138,
|
|
"normalizedEntropy": 0.13092569248012592,
|
|
"medianLatencyMs": 6921.807636260986,
|
|
"meanCompletionTokens": 322.2,
|
|
"meanReasoningTokens": 319.12
|
|
}
|
|
},
|
|
"meta": {
|
|
"tool": "llm-fingerprint-detector"
|
|
}
|
|
}
|