370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
531 lines
13 KiB
JSON
531 lines
13 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "MiniMax/MiniMax-M2.7",
|
|
"collectedAt": "2026-09-02T03:28:10.920Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"meta": {
|
|
"fusion": true,
|
|
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
|
|
"sourceDetector": "minimax_m27_reference.json",
|
|
"sourceExtra": "/tmp/mm_extra_cells2.json"
|
|
},
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"27": 1,
|
|
"42": 7,
|
|
"57": 1,
|
|
"58": 2,
|
|
"61": 1,
|
|
"73": 13
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.8535681581652277,
|
|
"normalizedEntropy": 0.2789898073076861,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 284.12,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 1,
|
|
"42": 10,
|
|
"45": 1,
|
|
"47": 4,
|
|
"63": 1,
|
|
"71": 1,
|
|
"73": 7
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.2090255736436504,
|
|
"normalizedEntropy": 0.33249147942778584,
|
|
"medianLatencyMs": 5043.271206999998,
|
|
"meanCompletionTokens": 197.36,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"green": 2,
|
|
"cyan": 2,
|
|
"blue": 9,
|
|
"turquoise": 1,
|
|
"mauve": 1,
|
|
"magenta": 6,
|
|
"azure": 1,
|
|
"teal": 2,
|
|
"crimson": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.6422921890824145,
|
|
"normalizedEntropy": 0.5384860611009273,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 181.52,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"elephant": 11,
|
|
"giraffe": 5,
|
|
"penguin": 4,
|
|
"lion": 1,
|
|
"otter": 1,
|
|
"zebra": 1,
|
|
"dog": 1,
|
|
"panda": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.337320658596841,
|
|
"normalizedEntropy": 0.41413540317194647,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 133.72,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"3": 1,
|
|
"5": 1,
|
|
"7": 22,
|
|
"9": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7195563653739032,
|
|
"normalizedEntropy": 0.21660804954849616,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 190.76,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"g": 4,
|
|
"k": 7,
|
|
"m": 6,
|
|
"q": 3,
|
|
"f": 1,
|
|
"x": 2,
|
|
"z": 1,
|
|
"r": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.647210311338979,
|
|
"normalizedEntropy": 0.5631835466631376,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 144.28,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"红": 8,
|
|
"蓝": 13,
|
|
"紫": 1,
|
|
"绿": 2,
|
|
"天蓝": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.6796275363413569,
|
|
"normalizedEntropy": 0.3422997728631977,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 177.52,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 23,
|
|
"tails": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4021791902022728,
|
|
"normalizedEntropy": 0.4021791902022728,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 191.12,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 22,
|
|
"42": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.5293608652873644,
|
|
"normalizedEntropy": 0.039837942232197415,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 201.08,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"cairo": 1,
|
|
"bangkok": 1,
|
|
"paris": 7,
|
|
"barcelona": 1,
|
|
"tokyo": 11,
|
|
"lagos": 1,
|
|
"denver": 1,
|
|
"sydney": 1,
|
|
"mumbai": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.3356468993981845,
|
|
"normalizedEntropy": 0.4138388401231415,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 163.6,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"5": 5,
|
|
"7": 20
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7219280948873623,
|
|
"normalizedEntropy": 0.2173220112736489,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 195.08,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 126.04,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"m": 4,
|
|
"k": 6,
|
|
"g": 7,
|
|
"x": 3,
|
|
"l": 1,
|
|
"a": 1,
|
|
"q": 2,
|
|
"u": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.6472103113389793,
|
|
"normalizedEntropy": 0.5631835466631377,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 192.84,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"猫": 15,
|
|
"熊猫": 3,
|
|
"猫头鹰": 1,
|
|
"大象": 2,
|
|
"狗": 2,
|
|
"企鹅": 1,
|
|
"老虎": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.949526332323075,
|
|
"normalizedEntropy": 0.3454245230158656,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 182.88,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"深圳": 1,
|
|
"北京": 10,
|
|
"东京": 12,
|
|
"上海": 1,
|
|
"杭州": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.5943029514736247,
|
|
"normalizedEntropy": 0.2824846873954918,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 150.12,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 189.36,
|
|
"meanReasoningTokens": 0
|
|
},
|
|
"binary-season:en": {
|
|
"cellId": "binary-season:en",
|
|
"counts": {
|
|
"summer": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": -0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-season:zh": {
|
|
"cellId": "binary-season:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:en": {
|
|
"cellId": "binary-pet:en",
|
|
"counts": {
|
|
"cat": 13,
|
|
"dog": 9
|
|
},
|
|
"validCount": 22,
|
|
"invalidCount": 3,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.0211917930491574,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:zh": {
|
|
"cellId": "binary-pet:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:en": {
|
|
"cellId": "binary-sea-mountain:en",
|
|
"counts": {
|
|
"sea": 16,
|
|
"mountain": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:zh": {
|
|
"cellId": "binary-sea-mountain:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:en": {
|
|
"cellId": "binary-tea-coffee:en",
|
|
"counts": {
|
|
"coffee": 7,
|
|
"tea": 18
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8554508105601306,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:zh": {
|
|
"cellId": "binary-tea-coffee:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:en": {
|
|
"cellId": "day-of-week:en",
|
|
"counts": {
|
|
"wednesday": 9,
|
|
"thursday": 4,
|
|
"monday": 8,
|
|
"friday": 2,
|
|
"tuesday": 1,
|
|
"saturday": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.142683189255492,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:zh": {
|
|
"cellId": "day-of-week:zh",
|
|
"counts": {
|
|
"monday": 12,
|
|
"wednesday": 9,
|
|
"thursday": 1,
|
|
"friday": 1,
|
|
"tuesday": 1,
|
|
"saturday": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.7819011889093375,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
}
|
|
}
|
|
} |