41 lines
869 B
Python
41 lines
869 B
Python
#!/usr/bin/env python3
|
|
"""
|
|
Lite evaluation suite: ~2-4h
|
|
Covers all 5 capability domains with 1-2 benchmarks each.
|
|
Use --limit 20 (default) for quick smoke testing.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
import run as run_module
|
|
|
|
run_module._multi_run_order = [
|
|
'aime24', # reasoning
|
|
'humaneval', # code
|
|
]
|
|
|
|
run_module._single_run_order = [
|
|
'gsm8k', # reasoning
|
|
'mmlu_pro', # knowledge
|
|
'simple_qa', # knowledge
|
|
'longbench_v2', # long-context
|
|
]
|
|
|
|
run_module._agent_order = [
|
|
'bfcl_v3', # tool/function calling
|
|
]
|
|
|
|
run_module.DATASETS = (
|
|
run_module._multi_run_order
|
|
+ run_module._single_run_order
|
|
+ run_module._agent_order
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
print('Lite benchmarks:', run_module.DATASETS)
|
|
run_module.main()
|
|
|