39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
"""Re-run only the review stage for bigcodebench using existing predictions.
|
|
|
|
The upstream `bigcodebench/bigcodebench-evaluate` image has an ENTRYPOINT that
|
|
runs the evaluator and exits, which kills the ms_enclave sandbox containers.
|
|
We use the locally built `bigcodebench-sandbox:latest` image (same libraries,
|
|
entrypoint dropped, runs `tail -f /dev/null`) and re-score the cached
|
|
predictions without regenerating them.
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
import run as run_mod
|
|
from evalscope import run_task
|
|
|
|
DATASET_NAME = 'bigcodebench'
|
|
BATCH_SIZE = 4
|
|
|
|
if DATASET_NAME not in run_mod.DATASET_CONFIGS:
|
|
raise SystemExit(f'{DATASET_NAME} not found in config')
|
|
|
|
ds_cfg = run_mod.DATASET_CONFIGS[DATASET_NAME]
|
|
task_cfg = run_mod.build_task_config(
|
|
dataset_name=DATASET_NAME,
|
|
ds_cfg=ds_cfg,
|
|
batch_size=BATCH_SIZE,
|
|
enable_thinking=run_mod.ENABLE_THINKING,
|
|
seed=run_mod.SEED,
|
|
run_idx=0,
|
|
)
|
|
task_cfg.rerun_review = True
|
|
|
|
print(f'Re-running review for {DATASET_NAME}')
|
|
print(f'Cache/work dir: {task_cfg.work_dir}')
|
|
print(f'Sandbox config: {task_cfg.sandbox.default_config}')
|
|
|
|
run_task(task_cfg)
|
|
print('Done.')
|