370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
522 lines
13 KiB
JSON
522 lines
13 KiB
JSON
{
|
|
"formatVersion": 1,
|
|
"protocol": "one-token/v1",
|
|
"model": "ZhipuAi/GLM-5.2",
|
|
"collectedAt": "2026-09-02T02:19:58.189Z",
|
|
"samplesPerCell": 25,
|
|
"postReasoning": false,
|
|
"meta": {
|
|
"fusion": true,
|
|
"note": "26-cell fp_fusion reference: 16 detector + 10 fp-only cells.",
|
|
"sourceDetector": "glm52_vectron_reference.json",
|
|
"sourceExtra": "/tmp/g52_extra_cells.json"
|
|
},
|
|
"cells": {
|
|
"random-number-1-100:en": {
|
|
"cellId": "random-number-1-100:en",
|
|
"counts": {
|
|
"42": 15,
|
|
"47": 1,
|
|
"57": 1,
|
|
"73": 8
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.3397218324096136,
|
|
"normalizedEntropy": 0.20164822870060348,
|
|
"medianLatencyMs": 2865.177502000006,
|
|
"meanCompletionTokens": 148.24,
|
|
"meanReasoningTokens": 145.32
|
|
},
|
|
"random-number-1-100:zh": {
|
|
"cellId": "random-number-1-100:zh",
|
|
"counts": {
|
|
"42": 20,
|
|
"57": 1,
|
|
"58": 1,
|
|
"73": 3
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.996118213778296,
|
|
"normalizedEntropy": 0.14993073078724659,
|
|
"medianLatencyMs": 3416.6582340000023,
|
|
"meanCompletionTokens": 217.12,
|
|
"meanReasoningTokens": 214.28
|
|
},
|
|
"random-color:en": {
|
|
"cellId": "random-color:en",
|
|
"counts": {
|
|
"teal": 5,
|
|
"blue": 5,
|
|
"magenta": 3,
|
|
"purple": 7,
|
|
"cerulean": 1,
|
|
"azure": 1,
|
|
"crimson": 1,
|
|
"green": 1,
|
|
"violet": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.738830073557111,
|
|
"normalizedEntropy": 0.558160003813466,
|
|
"medianLatencyMs": 2797.5234410000267,
|
|
"meanCompletionTokens": 153.6,
|
|
"meanReasoningTokens": 150.36
|
|
},
|
|
"random-animal:en": {
|
|
"cellId": "random-animal:en",
|
|
"counts": {
|
|
"kangaroo": 1,
|
|
"zebra": 3,
|
|
"elephant": 5,
|
|
"jaguar": 1,
|
|
"hippopotamus": 1,
|
|
"giraffe": 3,
|
|
"capybara": 4,
|
|
"penguin": 2,
|
|
"platypus": 3,
|
|
"fox": 1,
|
|
"tiger": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 3.2088840705376356,
|
|
"normalizedEntropy": 0.5685623379899973,
|
|
"medianLatencyMs": 3114.7327939999523,
|
|
"meanCompletionTokens": 168.24,
|
|
"meanReasoningTokens": 163.8
|
|
},
|
|
"random-number-1-10:en": {
|
|
"cellId": "random-number-1-10:en",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 207.88,
|
|
"meanReasoningTokens": 204.92
|
|
},
|
|
"random-letter:en": {
|
|
"cellId": "random-letter:en",
|
|
"counts": {
|
|
"r": 3,
|
|
"k": 10,
|
|
"q": 7,
|
|
"m": 3,
|
|
"g": 1,
|
|
"j": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.148634573470573,
|
|
"normalizedEntropy": 0.4571135260341782,
|
|
"medianLatencyMs": 2339.990761999972,
|
|
"meanCompletionTokens": 165.84,
|
|
"meanReasoningTokens": 162.92
|
|
},
|
|
"random-color:zh": {
|
|
"cellId": "random-color:zh",
|
|
"counts": {
|
|
"蓝": 15,
|
|
"青": 2,
|
|
"紫": 3,
|
|
"绿": 1,
|
|
"红": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.709526332323075,
|
|
"normalizedEntropy": 0.34839299939824137,
|
|
"medianLatencyMs": 4651.723928000021,
|
|
"meanCompletionTokens": 284.68,
|
|
"meanReasoningTokens": 281.68
|
|
},
|
|
"coin-flip:en": {
|
|
"cellId": "coin-flip:en",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 2993.0387099999934,
|
|
"meanCompletionTokens": 187.24,
|
|
"meanReasoningTokens": 183.96
|
|
},
|
|
"favorite-number:en": {
|
|
"cellId": "favorite-number:en",
|
|
"counts": {
|
|
"7": 12,
|
|
"42": 13
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9988455359952018,
|
|
"normalizedEntropy": 0.07516980073746855,
|
|
"medianLatencyMs": 4419.354362999991,
|
|
"meanCompletionTokens": 284.12,
|
|
"meanReasoningTokens": 281.16
|
|
},
|
|
"random-city:en": {
|
|
"cellId": "random-city:en",
|
|
"counts": {
|
|
"paris": 1,
|
|
"austin": 1,
|
|
"barcelona": 2,
|
|
"seattle": 2,
|
|
"tokyo": 8,
|
|
"stockholm": 1,
|
|
"oslo": 4,
|
|
"nairobi": 1,
|
|
"madrid": 1,
|
|
"berlin": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.883856189774724,
|
|
"normalizedEntropy": 0.5109726564258601,
|
|
"medianLatencyMs": 2799.2111550000263,
|
|
"meanCompletionTokens": 160.16,
|
|
"meanReasoningTokens": 156.52
|
|
},
|
|
"random-number-1-10:zh": {
|
|
"cellId": "random-number-1-10:zh",
|
|
"counts": {
|
|
"7": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 3109.8583069999004,
|
|
"meanCompletionTokens": 208.48,
|
|
"meanReasoningTokens": 205.72
|
|
},
|
|
"coin-flip:zh": {
|
|
"cellId": "coin-flip:zh",
|
|
"counts": {
|
|
"heads": 25
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0,
|
|
"normalizedEntropy": 0,
|
|
"medianLatencyMs": 3598.706825000001,
|
|
"meanCompletionTokens": 215.72,
|
|
"meanReasoningTokens": 212.8
|
|
},
|
|
"random-letter:zh": {
|
|
"cellId": "random-letter:zh",
|
|
"counts": {
|
|
"m": 8,
|
|
"j": 1,
|
|
"q": 7,
|
|
"k": 8,
|
|
"r": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.9377968115985953,
|
|
"normalizedEntropy": 0.41225862425589116,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": 235.4,
|
|
"meanReasoningTokens": 232.52
|
|
},
|
|
"random-animal:zh": {
|
|
"cellId": "random-animal:zh",
|
|
"counts": {
|
|
"猫": 20,
|
|
"狐狸": 2,
|
|
"老虎": 1,
|
|
"狼": 1,
|
|
"长颈鹿": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.106313713864835,
|
|
"normalizedEntropy": 0.196020890090928,
|
|
"medianLatencyMs": 4062.5094319999916,
|
|
"meanCompletionTokens": 249.48,
|
|
"meanReasoningTokens": 246.56
|
|
},
|
|
"random-city:zh": {
|
|
"cellId": "random-city:zh",
|
|
"counts": {
|
|
"伦敦": 1,
|
|
"东京": 6,
|
|
"北京": 8,
|
|
"巴黎": 4,
|
|
"柏林": 2,
|
|
"成都": 1,
|
|
"厦门": 1,
|
|
"杭州": 1,
|
|
"深圳": 1
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 2.663465189601647,
|
|
"normalizedEntropy": 0.47192293709169786,
|
|
"medianLatencyMs": 4759.383081000007,
|
|
"meanCompletionTokens": 261.96,
|
|
"meanReasoningTokens": 259
|
|
},
|
|
"favorite-number:zh": {
|
|
"cellId": "favorite-number:zh",
|
|
"counts": {
|
|
"0": 1,
|
|
"1": 1,
|
|
"7": 14,
|
|
"8": 7,
|
|
"42": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.6456780552463373,
|
|
"normalizedEntropy": 0.12384826986050242,
|
|
"medianLatencyMs": 6356.738842000021,
|
|
"meanCompletionTokens": 367.8,
|
|
"meanReasoningTokens": 364.96
|
|
},
|
|
"binary-season:en": {
|
|
"cellId": "binary-season:en",
|
|
"counts": {
|
|
"summer": 21,
|
|
"winter": 4
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.6343095546405662,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-season:zh": {
|
|
"cellId": "binary-season:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:en": {
|
|
"cellId": "binary-pet:en",
|
|
"counts": {
|
|
"cat": 3,
|
|
"dog": 20
|
|
},
|
|
"validCount": 23,
|
|
"invalidCount": 2,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.624609718596318,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-pet:zh": {
|
|
"cellId": "binary-pet:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:en": {
|
|
"cellId": "binary-sea-mountain:en",
|
|
"counts": {
|
|
"mountain": 14,
|
|
"sea": 11
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.9895875212220556,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-sea-mountain:zh": {
|
|
"cellId": "binary-sea-mountain:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:en": {
|
|
"cellId": "binary-tea-coffee:en",
|
|
"counts": {
|
|
"tea": 7,
|
|
"coffee": 18
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.8554508105601306,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"binary-tea-coffee:zh": {
|
|
"cellId": "binary-tea-coffee:zh",
|
|
"counts": {},
|
|
"validCount": 0,
|
|
"invalidCount": 25,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 0.0,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:en": {
|
|
"cellId": "day-of-week:en",
|
|
"counts": {
|
|
"wednesday": 9,
|
|
"tuesday": 1,
|
|
"thursday": 15
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.1585488318903812,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
},
|
|
"day-of-week:zh": {
|
|
"cellId": "day-of-week:zh",
|
|
"counts": {
|
|
"thursday": 3,
|
|
"wednesday": 18,
|
|
"monday": 2,
|
|
"friday": 2
|
|
},
|
|
"validCount": 25,
|
|
"invalidCount": 0,
|
|
"refusalCount": 0,
|
|
"emptyCount": 0,
|
|
"errorCount": 0,
|
|
"totalCount": 25,
|
|
"entropyBits": 1.291314688649721,
|
|
"normalizedEntropy": 0.0,
|
|
"medianLatencyMs": null,
|
|
"meanCompletionTokens": null,
|
|
"meanReasoningTokens": null
|
|
}
|
|
}
|
|
} |