46 lines
1.3 KiB
Python
46 lines
1.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Run a single benchmark for quick smoke tests.
|
|
|
|
Usage:
|
|
python bash/run_single.py bigcodebench --limit 1
|
|
python bash/run_single.py bfcl_v3 --limit 5 --model MODEL --api-url URL
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
import run as run_mod
|
|
from evalscope import run_task
|
|
|
|
if len(sys.argv) < 2:
|
|
print('Usage: python bash/run_single.py <dataset_name> [--limit N] [--model M] [--api-url U]')
|
|
sys.exit(1)
|
|
|
|
dataset_name = sys.argv[1]
|
|
limit = None
|
|
for i, arg in enumerate(sys.argv):
|
|
if arg == '--limit' and i + 1 < len(sys.argv):
|
|
v = sys.argv[i + 1]
|
|
limit = None if v.lower() in ('none', 'all') else int(v)
|
|
elif arg == '--model' and i + 1 < len(sys.argv):
|
|
run_mod.MODEL = sys.argv[i + 1]
|
|
elif arg == '--api-url' and i + 1 < len(sys.argv):
|
|
run_mod.API_URL = sys.argv[i + 1]
|
|
|
|
if dataset_name not in run_mod.DATASET_CONFIGS:
|
|
raise SystemExit(f'{dataset_name} not in config')
|
|
|
|
run_mod.LIMIT = limit
|
|
ds_cfg = run_mod.DATASET_CONFIGS[dataset_name]
|
|
task_cfg = run_mod.build_task_config(
|
|
dataset_name=dataset_name,
|
|
ds_cfg=ds_cfg,
|
|
batch_size=4,
|
|
enable_thinking=run_mod.ENABLE_THINKING,
|
|
seed=run_mod.SEED,
|
|
run_idx=0,
|
|
)
|
|
print(f'Running {dataset_name} with limit={limit}')
|
|
run_task(task_cfg)
|
|
print('Done.')
|