diff --git a/bash/collect_results.py b/bash/collect_results.py index f25cc57..f03a850 100644 --- a/bash/collect_results.py +++ b/bash/collect_results.py @@ -159,13 +159,10 @@ def find_report_candidates(seed_dir: Path, benchmark: str, model_name: str): """Return candidate report paths, accepting variant naming patterns.""" # The actual report filename may not match the directory name exactly # (e.g. hle_low directory holds hle.json). Try all .json files under - # the model directory. + # the reports directory. candidates = [ - seed_dir / 'reports' / model_name / f'{benchmark}.json', seed_dir / 'reports' / f'{benchmark}.json', ] - for p in (seed_dir / 'reports' / model_name).glob('*.json'): - candidates.append(p) for p in (seed_dir / 'reports').glob('*.json'): candidates.append(p) return candidates @@ -220,7 +217,7 @@ def find_all_predictions(output_dir: Path, benchmark: str, model_name: str): for seed_dir in sorted(bench_dir.iterdir()): if not seed_dir.is_dir(): continue - pred_dir = seed_dir / 'predictions' / model_name + pred_dir = seed_dir / 'predictions' if pred_dir.exists(): files.extend(sorted(pred_dir.rglob('*.jsonl'))) return files @@ -566,23 +563,21 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None, total_row[col] = np.nan df = pd.concat([df, pd.DataFrame([total_row], columns=OUTPUT_COLUMNS)], ignore_index=True) - summary_dir = output_dir / 'results' / safe_model - summary_dir.mkdir(parents=True, exist_ok=True) - if out_name is None: out_name = safe_model - csv_path = summary_dir / f'{out_name}.csv' - # Excel can optionally be written to a separate project-level results dir # so that the latest summary per model is easy to find independently of - # the per-run output directory. + # the per-run output directory. When a separate dir is given, put both + # CSV and Excel there. if excel_output_dir is not None: - xlsx_dir = Path(excel_output_dir) - xlsx_dir.mkdir(parents=True, exist_ok=True) - xlsx_path = xlsx_dir / f'{out_name}.xlsx' + summary_dir = Path(excel_output_dir) else: - xlsx_path = summary_dir / f'{out_name}.xlsx' + summary_dir = output_dir / 'results' / safe_model + summary_dir.mkdir(parents=True, exist_ok=True) + + csv_path = summary_dir / f'{out_name}.csv' + xlsx_path = summary_dir / f'{out_name}.xlsx' df.to_csv(csv_path, index=False, encoding='utf-8-sig') df.to_excel(xlsx_path, index=False) diff --git a/bash/run.py b/bash/run.py index a7c2815..303e4fa 100644 --- a/bash/run.py +++ b/bash/run.py @@ -210,7 +210,7 @@ def build_parser(): parser = argparse.ArgumentParser( description='Unified EvalScope benchmark runner', formatter_class=argparse.RawDescriptionHelpFormatter, - epilog='Suites: full, lite, mid, group1, group2, group3', + epilog='Suites: full, lite, mid, group1, group2, group3, official', ) # Model / API @@ -224,6 +224,8 @@ def build_parser(): help='Parent directory containing datasets/ subdir (default: %(default)s)') parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR, help='Output root directory (default: %(default)s)') + parser.add_argument('--folder-name', default=None, + help='Top-level output folder name; defaults to --model, or --model_THINKING when --thinking is enabled') parser.add_argument('--config', default=DEFAULT_CONFIG, help='YAML config path (default: %(default)s)') parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH, @@ -477,21 +479,20 @@ def build_task_config( # Main # ============================================================ -def write_summary(output_dir: str, model_name: str, +def write_summary(output_dir: str, model_name: str, folder_name: str, benchmark_names: list = None): """Re-aggregate results for the given benchmarks (or all on disk if None).""" try: - # Derive a safe file name from the model name - safe_name = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_') excel_output_dir = PROJECT_ROOT / 'results' + excel_output_dir.mkdir(parents=True, exist_ok=True) if benchmark_names is not None: collect_results_module.eval_benchmark( - benchmark_names, Path(output_dir), model_name, safe_name, + benchmark_names, Path(output_dir), model_name, folder_name, excel_output_dir=excel_output_dir, ) else: collect_results_module.collect_all( - Path(output_dir), model_name, safe_name, + Path(output_dir), model_name, folder_name, excel_output_dir=excel_output_dir, ) except Exception as e: @@ -499,7 +500,7 @@ def write_summary(output_dir: str, model_name: str, def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, - model_name: str, benchmark_names: list): + model_name: str, folder_name: str, benchmark_names: list): """Run a benchmark task and optionally refresh the summary table. ``benchmark_names`` is the running list of canonical benchmark names that @@ -526,18 +527,18 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, backup_after_run(output_dir, dataset_name, model_name, work_dir) if write_summary_flag: - write_summary(output_dir, model_name, benchmark_names=benchmark_names) + write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names) def backup_after_run(output_dir: str, benchmark: str, model_name: str, work_dir: Path): """Snapshot the just-finished run's perf summary and predictions.""" try: - report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + report_json = work_dir / 'reports' / f'{benchmark}.json' perf_backup_module.backup_perf_stats( Path(output_dir), benchmark, model_name, report_json, ) - predictions_dir = work_dir / 'predictions' / model_name + predictions_dir = work_dir / 'predictions' perf_backup_module.archive_predictions( Path(output_dir), benchmark, model_name, predictions_dir, ) @@ -552,8 +553,8 @@ def restore_before_run(output_dir: str, benchmark: str, model_name: str, the larger historical state instead of overwriting it. """ try: - predictions_dir = work_dir / 'predictions' / model_name - report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + predictions_dir = work_dir / 'predictions' + report_json = work_dir / 'reports' / f'{benchmark}.json' restored = perf_backup_module.restore_from_backup( Path(output_dir), benchmark, model_name, predictions_dir, report_json, @@ -616,12 +617,26 @@ def main(): dataset_configs = load_dataset_configs(args.config) + # Resolve top-level output folder name + if args.folder_name: + folder_name = args.folder_name + elif enable_thinking: + safe_model = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_') + folder_name = f'{safe_model}_THINKING' + else: + folder_name = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_') + model_output_dir = Path(args.output_dir) / folder_name + model_output_dir.mkdir(parents=True, exist_ok=True) + results_dir = PROJECT_ROOT / 'results' + results_dir.mkdir(parents=True, exist_ok=True) + print('=' * 60) print(f'Config: {args.config}') print(f'Model: {args.model}') print(f'API URL: {args.api_url}') print(f'Dataset Dir: {args.dataset_dir}') - print(f'Output Dir: {args.output_dir}') + print(f'Output Root: {args.output_dir}') + print(f'Output Folder: {folder_name}') print(f'Suite: {args.suite}') print(f'Limit: {limit if limit is not None else "ALL"}') print(f'Thinking: {enable_thinking}') @@ -651,13 +666,13 @@ def main(): ds_cfg = get_dataset_config(dataset_name) task_cfg = build_task_config( dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, + str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args, run_idx=run_idx, thinking_max_tokens_scale=args.thinking_max_tokens_scale, ) try: - run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, - benchmark_names=benchmark_names) + run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model, + folder_name=folder_name, benchmark_names=benchmark_names) except Exception as e: print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}') @@ -689,7 +704,7 @@ def main(): run_one(dataset_name, benchmark_names=benchmark_names) if args.write_summary: - write_summary(args.output_dir, args.model, benchmark_names=benchmark_names) + write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names) print('\nAll benchmarks done!') diff --git a/evalscope/evalscope/api/evaluator/cache.py b/evalscope/evalscope/api/evaluator/cache.py index e82cbb7..a454b2d 100644 --- a/evalscope/evalscope/api/evaluator/cache.py +++ b/evalscope/evalscope/api/evaluator/cache.py @@ -118,7 +118,7 @@ class CacheManager: Returns: Path to the prediction cache file """ - file_path = os.path.join(self.outputs.predictions_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl') + file_path = os.path.join(self.outputs.predictions_dir, f'{self.benchmark_name}_{subset}.jsonl') # Ensure the directory exists if self.outputs.is_make: os.makedirs(os.path.dirname(file_path), exist_ok=True) @@ -189,7 +189,7 @@ class CacheManager: Returns: Path to the review cache file """ - file_path = os.path.join(self.outputs.reviews_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl') + file_path = os.path.join(self.outputs.reviews_dir, f'{self.benchmark_name}_{subset}.jsonl') # Ensure the directory exists if self.outputs.is_make: os.makedirs(os.path.dirname(file_path), exist_ok=True) @@ -235,7 +235,7 @@ class CacheManager: Returns: Path to the reports directory for this model """ - report_path = os.path.join(self.outputs.reports_dir, self.model_name) + report_path = self.outputs.reports_dir # Ensure the directory exists if self.outputs.is_make: os.makedirs(report_path, exist_ok=True)