49 lines
961 B
Python
49 lines
961 B
Python
"""
|
|
Group 1: Code + Reasoning + short Knowledge + Agent (est. ~18-22h)
|
|
Run on machine 1.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
# Ensure we can import run.py from the same directory inside or outside Docker
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
import run as run_module
|
|
|
|
# Override dataset lists for group 1
|
|
run_module._multi_run_order = [
|
|
'humaneval',
|
|
'live_code_bench',
|
|
'aime26',
|
|
'aime24',
|
|
'aime25',
|
|
'gpqa_diamond',
|
|
'imo_answerbench',
|
|
'hmmt26',
|
|
]
|
|
|
|
run_module._single_run_order = [
|
|
'arc',
|
|
'winogrande',
|
|
'competition_math',
|
|
'gsm8k',
|
|
'hellaswag',
|
|
'bigcodebench',
|
|
]
|
|
|
|
run_module._agent_order = [
|
|
'bfcl_v3',
|
|
'tau2_bench',
|
|
]
|
|
|
|
run_module.DATASETS = (
|
|
run_module._multi_run_order
|
|
+ run_module._single_run_order
|
|
+ run_module._agent_order
|
|
)
|
|
|
|
if __name__ == '__main__':
|
|
print('Group 1 benchmarks:', run_module.DATASETS)
|
|
print('Estimated time: ~18-22h')
|
|
run_module.main()
|