f6ee6c7a8b
Adding config/sample_counts.yaml made TWO yamls in config/, and the auto-load rule was 'exactly one yaml' -- so every bench silently lost its repeats/temperature/max_tokens (humaneval ran once instead of 3). Manifest renamed to .json, and the rule hardened: a lone yaml still wins, otherwise default.yaml wins explicitly. Verified: aime 12x + humaneval 3x repeats active again. Co-Authored-By: Claude <noreply@anthropic.com>
29 lines
526 B
JSON
29 lines
526 B
JSON
{
|
|
"humaneval": 164,
|
|
"aime24": 30,
|
|
"aime25": 30,
|
|
"aime26": 30,
|
|
"hmmt26": 30,
|
|
"gpqa_diamond": 198,
|
|
"mmlu": 285,
|
|
"mmlu_pro": 12032,
|
|
"cmmlu": 11528,
|
|
"gsm8k": 1319,
|
|
"arc": 2376,
|
|
"hellaswag": 10042,
|
|
"winogrande": 1267,
|
|
"drop": 9535,
|
|
"longbench_v2": 503,
|
|
"live_code_bench": 1055,
|
|
"bigcodebench": 1140,
|
|
"trivia_qa": 17944,
|
|
"simple_qa": 4326,
|
|
"hle": 2500,
|
|
"imo_answerbench": 89,
|
|
"openai_mrcr": 1300,
|
|
"bfcl_v3": 2000,
|
|
"general_fc": 400,
|
|
"tau2_bench": 184,
|
|
"competition_math": 5000,
|
|
"swe_bench_verified": 500
|
|
} |