75 lines
1.4 KiB
Python
75 lines
1.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Full evaluation suite: ~3-5 days
|
|
Runs all benchmarks with full datasets and multiple runs for stability.
|
|
Use --limit none (default) for the complete evaluation.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
import run as run_module
|
|
|
|
# Use the default full configuration from run.py
|
|
# (already ordered by estimated runtime)
|
|
run_module._multi_run_order = [
|
|
'humaneval',
|
|
'live_code_bench',
|
|
'aime26',
|
|
'aime24',
|
|
'aime25',
|
|
'gpqa_diamond',
|
|
'imo_answerbench',
|
|
'hmmt26',
|
|
]
|
|
|
|
run_module._single_run_order = [
|
|
'arc',
|
|
'bfcl_v3',
|
|
'winogrande',
|
|
'competition_math',
|
|
'gsm8k',
|
|
'hellaswag',
|
|
'bigcodebench',
|
|
'drop',
|
|
'bbh',
|
|
'openai_mrcr',
|
|
'longbench_v2',
|
|
'mmlu',
|
|
'cmmlu',
|
|
'super_gpqa',
|
|
'simple_qa',
|
|
'mmlu_pro',
|
|
'hle',
|
|
'trivia_qa',
|
|
]
|
|
|
|
run_module._agent_order = [
|
|
'tau2_bench',
|
|
'general_fc',
|
|
]
|
|
|
|
run_module.DATASETS = (
|
|
run_module._multi_run_order
|
|
+ run_module._single_run_order
|
|
+ run_module._agent_order
|
|
)
|
|
|
|
# Full: many runs for statistical stability
|
|
run_module.MULTI_RUN_CONFIG = {
|
|
'aime24': 12,
|
|
'aime25': 12,
|
|
'aime26': 12,
|
|
'hmmt26': 12,
|
|
'live_code_bench': 5,
|
|
'imo_answerbench': 4,
|
|
'humaneval': 3,
|
|
'gpqa_diamond': 2,
|
|
}
|
|
|
|
if __name__ == '__main__':
|
|
print('Full benchmarks:', run_module.DATASETS)
|
|
print('Estimated time: ~3-5 days with --limit none')
|
|
run_module.main()
|