370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
517 lines
13 KiB
JSON
517 lines
13 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "ZhipuAi/GLM-5.3",
|
|
"collectedAt": "2026-09-01T04:07:46.932Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"meta": {
|
|
"fusion": true,
|
|
"note": "26-cell fp_fusion reference: 16 detector cells + 10 fp_fusion-only cells (binary preferences + day-of-week).",
|
|
"sourceDetector": "glm53_reference.json",
|
|
"sourceExtra": "/tmp/glm53_extra_cells.json"
|
|
},
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"37": 1,
|
|
"42": 2,
|
|
"47": 16,
|
|
"57": 2,
|
|
"67": 1,
|
|
"73": 2,
|
|
"83": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.8438561897747248,
|
|
"normalizedEntropy": 0.27752801040644515,
|
|
"medianLatencyMs": 3048.427018000046,
|
|
"meanCompletionTokens": 75.44,
|
|
"meanReasoningTokens": 72.28
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"37": 1,
|
|
"42": 7,
|
|
"47": 11,
|
|
"57": 1,
|
|
"63": 1,
|
|
"68": 1,
|
|
"73": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.145451399311646,
|
|
"normalizedEntropy": 0.3229226127160336,
|
|
"medianLatencyMs": 3012.2534959999903,
|
|
"meanCompletionTokens": 59.72,
|
|
"meanReasoningTokens": 56.44
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"teal": 16,
|
|
"turquoise": 6,
|
|
"periwinkle": 1,
|
|
"indigo": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.3834651896016472,
|
|
"normalizedEntropy": 0.28194335346294375,
|
|
"medianLatencyMs": 3771.320187999998,
|
|
"meanCompletionTokens": 80.44,
|
|
"meanReasoningTokens": 76.32
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"capybara": 13,
|
|
"axolotl": 2,
|
|
"hedgehog": 1,
|
|
"pangolin": 4,
|
|
"platypus": 3,
|
|
"okapi": 1,
|
|
"narwhal": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.1294320362548183,
|
|
"normalizedEntropy": 0.37730090290266854,
|
|
"medianLatencyMs": 4167.4443130000145,
|
|
"meanCompletionTokens": 76.16,
|
|
"meanReasoningTokens": 71
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"4": 4,
|
|
"7": 21
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6343095546405662,
|
|
"normalizedEntropy": 0.19094620248307148,
|
|
"medianLatencyMs": 2412.1367320000136,
|
|
"meanCompletionTokens": 65.76,
|
|
"meanReasoningTokens": 62.36
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"k": 5,
|
|
"r": 6,
|
|
"q": 9,
|
|
"m": 2,
|
|
"j": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.1477110700184037,
|
|
"normalizedEntropy": 0.45691705431928625,
|
|
"medianLatencyMs": 3700.4820349999936,
|
|
"meanCompletionTokens": 69.84,
|
|
"meanReasoningTokens": 66.8
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 16,
|
|
"青": 6,
|
|
"靛蓝": 1,
|
|
"紫": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.3834651896016472,
|
|
"normalizedEntropy": 0.28194335346294375,
|
|
"medianLatencyMs": 3500.8726499999757,
|
|
"meanCompletionTokens": 70.04,
|
|
"meanReasoningTokens": 66.04
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 23,
|
|
"tails": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.4021791902022728,
|
|
"normalizedEntropy": 0.4021791902022728,
|
|
"medianLatencyMs": 3116.6536569999953,
|
|
"meanCompletionTokens": 75.84,
|
|
"meanReasoningTokens": 72.04
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 10,
|
|
"42": 15
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9709505944546686,
|
|
"normalizedEntropy": 0.07307051999623161,
|
|
"medianLatencyMs": 7386.655828999996,
|
|
"meanCompletionTokens": 225.4,
|
|
"meanReasoningTokens": 222.08
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"lisbon": 6,
|
|
"osaka": 2,
|
|
"nairobi": 2,
|
|
"barcelona": 4,
|
|
"copenhagen": 1,
|
|
"helsinki": 1,
|
|
"kyoto": 5,
|
|
"valencia": 1,
|
|
"oslo": 2,
|
|
"budapest": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.999079570624174,
|
|
"normalizedEntropy": 0.5313883752137,
|
|
"medianLatencyMs": 3566.0539570000255,
|
|
"meanCompletionTokens": 64.68,
|
|
"meanReasoningTokens": 60.4
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 2703.40669600002,
|
|
"meanCompletionTokens": 54.96,
|
|
"meanReasoningTokens": 51.52
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 24,
|
|
"tails": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.24229218908241482,
|
|
"normalizedEntropy": 0.24229218908241482,
|
|
"medianLatencyMs": 3555.5682660000166,
|
|
"meanCompletionTokens": 78.72,
|
|
"meanReasoningTokens": 75.28
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"k": 8,
|
|
"q": 11,
|
|
"m": 4,
|
|
"r": 1,
|
|
"g": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.841706277574314,
|
|
"normalizedEntropy": 0.3918157423583902,
|
|
"medianLatencyMs": 3158.31832999998,
|
|
"meanCompletionTokens": 65.72,
|
|
"meanReasoningTokens": 62.56
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"猫": 10,
|
|
"水獭": 4,
|
|
"斑马": 2,
|
|
"企鹅": 3,
|
|
"水豚": 4,
|
|
"鸭嘴兽": 1,
|
|
"袋鼠": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.4048894517332404,
|
|
"normalizedEntropy": 0.426107500061803,
|
|
"medianLatencyMs": 5038.485356999969,
|
|
"meanCompletionTokens": 105.2,
|
|
"meanReasoningTokens": 98.92
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"布拉格": 2,
|
|
"北京": 4,
|
|
"京都": 2,
|
|
"成都": 5,
|
|
"巴黎": 4,
|
|
"杭州": 1,
|
|
"里斯本": 4,
|
|
"上海": 1,
|
|
"东京": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.9794705707972517,
|
|
"normalizedEntropy": 0.5279139777153283,
|
|
"medianLatencyMs": 4077.9174099999946,
|
|
"meanCompletionTokens": 97.32,
|
|
"meanReasoningTokens": 93.76
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"7": 12,
|
|
"42": 13
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9988455359952018,
|
|
"normalizedEntropy": 0.07516980073746855,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 171.16,
|
|
"meanReasoningTokens": 169.16
|
|
},
|
|
"binary-season:en": {
|
|
"cellId": "binary-season:en",
|
|
"counts": {
|
|
"summer": 21,
|
|
"winter": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6343095546405662,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-season:zh": {
|
|
"cellId": "binary-season:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:en": {
|
|
"cellId": "binary-pet:en",
|
|
"counts": {
|
|
"cat": 16,
|
|
"dog": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:zh": {
|
|
"cellId": "binary-pet:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:en": {
|
|
"cellId": "binary-sea-mountain:en",
|
|
"counts": {
|
|
"mountain": 16,
|
|
"sea": 9
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:zh": {
|
|
"cellId": "binary-sea-mountain:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:en": {
|
|
"cellId": "binary-tea-coffee:en",
|
|
"counts": {
|
|
"tea": 19,
|
|
"coffee": 6
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.7950402793845223,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:zh": {
|
|
"cellId": "binary-tea-coffee:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:en": {
|
|
"cellId": "day-of-week:en",
|
|
"counts": {
|
|
"wednesday": 9,
|
|
"thursday": 16
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9426831892554922,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:zh": {
|
|
"cellId": "day-of-week:zh",
|
|
"counts": {
|
|
"thursday": 9,
|
|
"wednesday": 14,
|
|
"friday": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.290564432903234,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
}
|
|
}
|
|
} |