36 lines
892 B
Python
36 lines
892 B
Python
"""
|
|
Group 3: Long-running Knowledge benchmarks (est. ~35-40h)
|
|
Run on machine 3.
|
|
Note: trivia_qa and hle are inherently slow; consider using --limit to control time.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
# Ensure we can import run.py from the same directory inside or outside Docker
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
import run as run_module
|
|
|
|
# Override dataset lists for group 3
|
|
run_module._multi_run_order = []
|
|
|
|
run_module._single_run_order = [
|
|
'mmlu_pro',
|
|
'hle',
|
|
'trivia_qa',
|
|
]
|
|
|
|
run_module._agent_order = []
|
|
|
|
run_module.DATASETS = (
|
|
run_module._multi_run_order
|
|
+ run_module._single_run_order
|
|
+ run_module._agent_order
|
|
)
|
|
|
|
if __name__ == '__main__':
|
|
print('Group 3 benchmarks:', run_module.DATASETS)
|
|
print('Estimated time: ~35-40h (trivia_qa and hle are slow)')
|
|
print('Tip: use --limit 500 to cap runtime if needed')
|
|
run_module.main()
|