feat: restructure output layout and add --folder-name

- Add --folder-name CLI arg; defaults to safe model name, or
  {model}_THINKING when --thinking is enabled.
- Move benchmark outputs under output/{folder_name}/{benchmark}/...
- Remove redundant {model_name} subfolder from predictions/reports/reviews.
- Update collect_results.py to write both CSV and Excel to
  results/{folder_name}.csv/xlsx.
- Update perf backup/restore paths to match the new layout.
- Patch evalscope CacheManager to drop model_name from cache paths.

bash/collect_results.py

@@ -159,13 +159,10 @@ def find_report_candidates(seed_dir: Path, benchmark: str, model_name: str):
     """Return candidate report paths, accepting variant naming patterns."""
     # The actual report filename may not match the directory name exactly
     # (e.g. hle_low directory holds hle.json). Try all .json files under
-    # the model directory.
+    # the reports directory.
     candidates = [
-        seed_dir / 'reports' / model_name / f'{benchmark}.json',
         seed_dir / 'reports' / f'{benchmark}.json',
     ]
-    for p in (seed_dir / 'reports' / model_name).glob('*.json'):
-        candidates.append(p)
     for p in (seed_dir / 'reports').glob('*.json'):
         candidates.append(p)
     return candidates
@@ -220,7 +217,7 @@ def find_all_predictions(output_dir: Path, benchmark: str, model_name: str):
     for seed_dir in sorted(bench_dir.iterdir()):
         if not seed_dir.is_dir():
             continue
-        pred_dir = seed_dir / 'predictions' / model_name
+        pred_dir = seed_dir / 'predictions'
         if pred_dir.exists():
             files.extend(sorted(pred_dir.rglob('*.jsonl')))
     return files
@@ -566,23 +563,21 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None,
             total_row[col] = np.nan
     df = pd.concat([df, pd.DataFrame([total_row], columns=OUTPUT_COLUMNS)], ignore_index=True)
 
-    summary_dir = output_dir / 'results' / safe_model
-    summary_dir.mkdir(parents=True, exist_ok=True)
-
     if out_name is None:
         out_name = safe_model
 
-    csv_path = summary_dir / f'{out_name}.csv'
-
     # Excel can optionally be written to a separate project-level results dir
     # so that the latest summary per model is easy to find independently of
-    # the per-run output directory.
+    # the per-run output directory. When a separate dir is given, put both
+    # CSV and Excel there.
     if excel_output_dir is not None:
-        xlsx_dir = Path(excel_output_dir)
+        summary_dir = Path(excel_output_dir)
-        xlsx_dir.mkdir(parents=True, exist_ok=True)
-        xlsx_path = xlsx_dir / f'{out_name}.xlsx'
     else:
-        xlsx_path = summary_dir / f'{out_name}.xlsx'
+        summary_dir = output_dir / 'results' / safe_model
+    summary_dir.mkdir(parents=True, exist_ok=True)
+
+    csv_path = summary_dir / f'{out_name}.csv'
+    xlsx_path = summary_dir / f'{out_name}.xlsx'
 
     df.to_csv(csv_path, index=False, encoding='utf-8-sig')
     df.to_excel(xlsx_path, index=False)

bash/run.py

@@ -210,7 +210,7 @@ def build_parser():
     parser = argparse.ArgumentParser(
         description='Unified EvalScope benchmark runner',
         formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog='Suites: full, lite, mid, group1, group2, group3',
+        epilog='Suites: full, lite, mid, group1, group2, group3, official',
     )
 
     # Model / API
@@ -224,6 +224,8 @@ def build_parser():
                         help='Parent directory containing datasets/ subdir (default: %(default)s)')
     parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR,
                         help='Output root directory (default: %(default)s)')
+    parser.add_argument('--folder-name', default=None,
+                        help='Top-level output folder name; defaults to --model, or --model_THINKING when --thinking is enabled')
     parser.add_argument('--config', default=DEFAULT_CONFIG,
                         help='YAML config path (default: %(default)s)')
     parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH,
@@ -477,21 +479,20 @@ def build_task_config(
 # Main
 # ============================================================
 
-def write_summary(output_dir: str, model_name: str,
+def write_summary(output_dir: str, model_name: str, folder_name: str,
                   benchmark_names: list = None):
     """Re-aggregate results for the given benchmarks (or all on disk if None)."""
     try:
-        # Derive a safe file name from the model name
-        safe_name = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_')
         excel_output_dir = PROJECT_ROOT / 'results'
+        excel_output_dir.mkdir(parents=True, exist_ok=True)
         if benchmark_names is not None:
             collect_results_module.eval_benchmark(
-                benchmark_names, Path(output_dir), model_name, safe_name,
+                benchmark_names, Path(output_dir), model_name, folder_name,
                 excel_output_dir=excel_output_dir,
             )
         else:
             collect_results_module.collect_all(
-                Path(output_dir), model_name, safe_name,
+                Path(output_dir), model_name, folder_name,
                 excel_output_dir=excel_output_dir,
             )
     except Exception as e:
@@ -499,7 +500,7 @@ def write_summary(output_dir: str, model_name: str,
 
 
 def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
-                       model_name: str, benchmark_names: list):
+                       model_name: str, folder_name: str, benchmark_names: list):
     """Run a benchmark task and optionally refresh the summary table.
 
     ``benchmark_names`` is the running list of canonical benchmark names that
@@ -526,18 +527,18 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
 
     backup_after_run(output_dir, dataset_name, model_name, work_dir)
     if write_summary_flag:
-        write_summary(output_dir, model_name, benchmark_names=benchmark_names)
+        write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
 
 
 def backup_after_run(output_dir: str, benchmark: str, model_name: str,
                      work_dir: Path):
     """Snapshot the just-finished run's perf summary and predictions."""
     try:
-        report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
+        report_json = work_dir / 'reports' / f'{benchmark}.json'
         perf_backup_module.backup_perf_stats(
             Path(output_dir), benchmark, model_name, report_json,
         )
-        predictions_dir = work_dir / 'predictions' / model_name
+        predictions_dir = work_dir / 'predictions'
         perf_backup_module.archive_predictions(
             Path(output_dir), benchmark, model_name, predictions_dir,
         )
@@ -552,8 +553,8 @@ def restore_before_run(output_dir: str, benchmark: str, model_name: str,
     the larger historical state instead of overwriting it.
     """
     try:
-        predictions_dir = work_dir / 'predictions' / model_name
+        predictions_dir = work_dir / 'predictions'
-        report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
+        report_json = work_dir / 'reports' / f'{benchmark}.json'
         restored = perf_backup_module.restore_from_backup(
             Path(output_dir), benchmark, model_name,
             predictions_dir, report_json,
@@ -616,12 +617,26 @@ def main():
 
     dataset_configs = load_dataset_configs(args.config)
 
+    # Resolve top-level output folder name
+    if args.folder_name:
+        folder_name = args.folder_name
+    elif enable_thinking:
+        safe_model = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_')
+        folder_name = f'{safe_model}_THINKING'
+    else:
+        folder_name = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_')
+    model_output_dir = Path(args.output_dir) / folder_name
+    model_output_dir.mkdir(parents=True, exist_ok=True)
+    results_dir = PROJECT_ROOT / 'results'
+    results_dir.mkdir(parents=True, exist_ok=True)
+
     print('=' * 60)
     print(f'Config: {args.config}')
     print(f'Model: {args.model}')
     print(f'API URL: {args.api_url}')
     print(f'Dataset Dir: {args.dataset_dir}')
-    print(f'Output Dir: {args.output_dir}')
+    print(f'Output Root: {args.output_dir}')
+    print(f'Output Folder: {folder_name}')
     print(f'Suite: {args.suite}')
     print(f'Limit: {limit if limit is not None else "ALL"}')
     print(f'Thinking: {enable_thinking}')
@@ -651,13 +666,13 @@ def main():
         ds_cfg = get_dataset_config(dataset_name)
         task_cfg = build_task_config(
             dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
-            args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
+            str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args,
             run_idx=run_idx,
             thinking_max_tokens_scale=args.thinking_max_tokens_scale,
         )
         try:
-            run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model,
+            run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model,
-                              benchmark_names=benchmark_names)
+                              folder_name=folder_name, benchmark_names=benchmark_names)
         except Exception as e:
             print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
 
@@ -689,7 +704,7 @@ def main():
         run_one(dataset_name, benchmark_names=benchmark_names)
 
     if args.write_summary:
-        write_summary(args.output_dir, args.model, benchmark_names=benchmark_names)
+        write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
     print('\nAll benchmarks done!')
 
 

evalscope/evalscope/api/evaluator/cache.py

@@ -118,7 +118,7 @@ class CacheManager:
         Returns:
             Path to the prediction cache file
         """
-        file_path = os.path.join(self.outputs.predictions_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl')
+        file_path = os.path.join(self.outputs.predictions_dir, f'{self.benchmark_name}_{subset}.jsonl')
         # Ensure the directory exists
         if self.outputs.is_make:
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -189,7 +189,7 @@ class CacheManager:
         Returns:
             Path to the review cache file
         """
-        file_path = os.path.join(self.outputs.reviews_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl')
+        file_path = os.path.join(self.outputs.reviews_dir, f'{self.benchmark_name}_{subset}.jsonl')
         # Ensure the directory exists
         if self.outputs.is_make:
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -235,7 +235,7 @@ class CacheManager:
         Returns:
             Path to the reports directory for this model
         """
-        report_path = os.path.join(self.outputs.reports_dir, self.model_name)
+        report_path = self.outputs.reports_dir
         # Ensure the directory exists
         if self.outputs.is_make:
             os.makedirs(report_path, exist_ok=True)