#!/usr/bin/env python3
"""
Full evaluation suite: ~3-5 days
Runs all benchmarks with full datasets and multiple runs for stability.
Use --limit none (default) for the complete evaluation.
"""
from pathlib import Path
import sys

sys.path.insert(0, str(Path(__file__).parent))

import run as run_module

# Use the default full configuration from run.py
# (already ordered by estimated runtime)
run_module._multi_run_order = [
    'humaneval',
    'live_code_bench',
    'aime26',
    'aime24',
    'aime25',
    'gpqa_diamond',
    'imo_answerbench',
    'hmmt26',
]

run_module._single_run_order = [
    'arc',
    'bfcl_v3',
    'winogrande',
    'competition_math',
    'gsm8k',
    'hellaswag',
    'bigcodebench',
    'drop',
    'bbh',
    'openai_mrcr',
    'longbench_v2',
    'mmlu',
    'cmmlu',
    'super_gpqa',
    'simple_qa',
    'mmlu_pro',
    'hle',
    'trivia_qa',
]

run_module._agent_order = [
    'tau2_bench',
    'general_fc',
]

run_module.DATASETS = (
    run_module._multi_run_order
    + run_module._single_run_order
    + run_module._agent_order
)

# Full: many runs for statistical stability
run_module.MULTI_RUN_CONFIG = {
    'aime24': 12,
    'aime25': 12,
    'aime26': 12,
    'hmmt26': 12,
    'live_code_bench': 5,
    'imo_answerbench': 4,
    'humaneval': 3,
    'gpqa_diamond': 2,
}

if __name__ == '__main__':
    print('Full benchmarks:', run_module.DATASETS)
    print('Estimated time: ~3-5 days with --limit none')
    run_module.main()