sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: restructure output layout and add --folder-name

Browse Source 
- Add --folder-name CLI arg; defaults to safe model name, or
  {model}_THINKING when --thinking is enabled.
- Move benchmark outputs under output/{folder_name}/{benchmark}/...
- Remove redundant {model_name} subfolder from predictions/reports/reviews.
- Update collect_results.py to write both CSV and Excel to
  results/{folder_name}.csv/xlsx.
- Update perf backup/restore paths to match the new layout.
- Patch evalscope CacheManager to drop model_name from cache paths.
This commit is contained in:
sora 2026-07-23 03:06:52 +00:00
parent 1308ddd251
commit e26fa0fe44
3 changed files with 45 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
bash/collect_results.py
View File

@ -159,13 +159,10 @@ def find_report_candidates(seed_dir: Path, benchmark: str, model_name: str):
"""Return candidate report paths, accepting variant naming patterns."""
# The actual report filename may not match the directory name exactly
# (e.g. hle_low directory holds hle.json). Try all .json files under
# the model directory.
# the reports directory.
candidates = [
seed_dir / 'reports' / model_name / f'{benchmark}.json',
seed_dir / 'reports' / f'{benchmark}.json',
]
for p in (seed_dir / 'reports' / model_name).glob('*.json'):
candidates.append(p)
for p in (seed_dir / 'reports').glob('*.json'):
candidates.append(p)
return candidates
@ -220,7 +217,7 @@ def find_all_predictions(output_dir: Path, benchmark: str, model_name: str):
for seed_dir in sorted(bench_dir.iterdir()):
if not seed_dir.is_dir():
continue
pred_dir = seed_dir / 'predictions' / model_name
pred_dir = seed_dir / 'predictions'
if pred_dir.exists():
files.extend(sorted(pred_dir.rglob('*.jsonl')))
return files
@ -566,22 +563,20 @@ def collect_all(output_dir: Path, model_name: str, out_name: str = None,
total_row[col] = np.nan
df = pd.concat([df, pd.DataFrame([total_row], columns=OUTPUT_COLUMNS)], ignore_index=True)
summary_dir = output_dir / 'results' / safe_model
summary_dir.mkdir(parents=True, exist_ok=True)
if out_name is None:
out_name = safe_model
csv_path = summary_dir / f'{out_name}.csv'
# Excel can optionally be written to a separate project-level results dir
# so that the latest summary per model is easy to find independently of
# the per-run output directory.
# the per-run output directory. When a separate dir is given, put both
# CSV and Excel there.
if excel_output_dir is not None:
xlsx_dir = Path(excel_output_dir)
xlsx_dir.mkdir(parents=True, exist_ok=True)
xlsx_path = xlsx_dir / f'{out_name}.xlsx'
summary_dir = Path(excel_output_dir)
else:
summary_dir = output_dir / 'results' / safe_model
summary_dir.mkdir(parents=True, exist_ok=True)
csv_path = summary_dir / f'{out_name}.csv'
xlsx_path = summary_dir / f'{out_name}.xlsx'
df.to_csv(csv_path, index=False, encoding='utf-8-sig')

49
bash/run.py
View File

@ -210,7 +210,7 @@ def build_parser():
parser = argparse.ArgumentParser(
description='Unified EvalScope benchmark runner',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='Suites: full, lite, mid, group1, group2, group3',
epilog='Suites: full, lite, mid, group1, group2, group3, official',
)
# Model / API
@ -224,6 +224,8 @@ def build_parser():
help='Parent directory containing datasets/ subdir (default: %(default)s)')
parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR,
help='Output root directory (default: %(default)s)')
parser.add_argument('--folder-name', default=None,
help='Top-level output folder name; defaults to --model, or --model_THINKING when --thinking is enabled')
parser.add_argument('--config', default=DEFAULT_CONFIG,
help='YAML config path (default: %(default)s)')
parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH,
@ -477,21 +479,20 @@ def build_task_config(
# Main
# ============================================================
def write_summary(output_dir: str, model_name: str,
def write_summary(output_dir: str, model_name: str, folder_name: str,
benchmark_names: list = None):
"""Re-aggregate results for the given benchmarks (or all on disk if None)."""
try:
# Derive a safe file name from the model name
safe_name = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_')
excel_output_dir = PROJECT_ROOT / 'results'
excel_output_dir.mkdir(parents=True, exist_ok=True)
if benchmark_names is not None:
collect_results_module.eval_benchmark(
benchmark_names, Path(output_dir), model_name, safe_name,
benchmark_names, Path(output_dir), model_name, folder_name,
excel_output_dir=excel_output_dir,
)
else:
collect_results_module.collect_all(
Path(output_dir), model_name, safe_name,
Path(output_dir), model_name, folder_name,
excel_output_dir=excel_output_dir,
)
except Exception as e:
@ -499,7 +500,7 @@ def write_summary(output_dir: str, model_name: str,
def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
model_name: str, benchmark_names: list):
model_name: str, folder_name: str, benchmark_names: list):
"""Run a benchmark task and optionally refresh the summary table.
``benchmark_names`` is the running list of canonical benchmark names that
@ -526,18 +527,18 @@ def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str,
backup_after_run(output_dir, dataset_name, model_name, work_dir)
if write_summary_flag:
write_summary(output_dir, model_name, benchmark_names=benchmark_names)
write_summary(output_dir, model_name, folder_name, benchmark_names=benchmark_names)
def backup_after_run(output_dir: str, benchmark: str, model_name: str,
work_dir: Path):
"""Snapshot the just-finished run's perf summary and predictions."""
try:
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
report_json = work_dir / 'reports' / f'{benchmark}.json'
perf_backup_module.backup_perf_stats(
Path(output_dir), benchmark, model_name, report_json,
)
predictions_dir = work_dir / 'predictions' / model_name
predictions_dir = work_dir / 'predictions'
perf_backup_module.archive_predictions(
Path(output_dir), benchmark, model_name, predictions_dir,
)
@ -552,8 +553,8 @@ def restore_before_run(output_dir: str, benchmark: str, model_name: str,
the larger historical state instead of overwriting it.
"""
try:
predictions_dir = work_dir / 'predictions' / model_name
report_json = work_dir / 'reports' / model_name / f'{benchmark}.json'
predictions_dir = work_dir / 'predictions'
report_json = work_dir / 'reports' / f'{benchmark}.json'
restored = perf_backup_module.restore_from_backup(
Path(output_dir), benchmark, model_name,
predictions_dir, report_json,
@ -616,12 +617,26 @@ def main():
dataset_configs = load_dataset_configs(args.config)
# Resolve top-level output folder name
if args.folder_name:
folder_name = args.folder_name
elif enable_thinking:
safe_model = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_')
folder_name = f'{safe_model}_THINKING'
else:
folder_name = args.model.replace('/', '_').replace('\\', '_').replace(' ', '_')
model_output_dir = Path(args.output_dir) / folder_name
model_output_dir.mkdir(parents=True, exist_ok=True)
results_dir = PROJECT_ROOT / 'results'
results_dir.mkdir(parents=True, exist_ok=True)
print('=' * 60)
print(f'Config: {args.config}')
print(f'Model: {args.model}')
print(f'API URL: {args.api_url}')
print(f'Dataset Dir: {args.dataset_dir}')
print(f'Output Dir: {args.output_dir}')
print(f'Output Root: {args.output_dir}')
print(f'Output Folder: {folder_name}')
print(f'Suite: {args.suite}')
print(f'Limit: {limit if limit is not None else "ALL"}')
print(f'Thinking: {enable_thinking}')
@ -651,13 +666,13 @@ def main():
ds_cfg = get_dataset_config(dataset_name)
task_cfg = build_task_config(
dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit,
args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args,
str(model_output_dir), args.model, args.api_url, args.dataset_dir, judge_model_args,
run_idx=run_idx,
thinking_max_tokens_scale=args.thinking_max_tokens_scale,
)
try:
run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model,
benchmark_names=benchmark_names)
run_and_summarize(task_cfg, args.write_summary, str(model_output_dir), args.model,
folder_name=folder_name, benchmark_names=benchmark_names)
except Exception as e:
print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}')
@ -689,7 +704,7 @@ def main():
run_one(dataset_name, benchmark_names=benchmark_names)
if args.write_summary:
write_summary(args.output_dir, args.model, benchmark_names=benchmark_names)
write_summary(str(model_output_dir), args.model, folder_name, benchmark_names=benchmark_names)
print('\nAll benchmarks done!')

6
evalscope/evalscope/api/evaluator/cache.py
View File

@ -118,7 +118,7 @@ class CacheManager:
Returns:
Path to the prediction cache file
"""
file_path = os.path.join(self.outputs.predictions_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl')
file_path = os.path.join(self.outputs.predictions_dir, f'{self.benchmark_name}_{subset}.jsonl')
# Ensure the directory exists
if self.outputs.is_make:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
@ -189,7 +189,7 @@ class CacheManager:
Returns:
Path to the review cache file
"""
file_path = os.path.join(self.outputs.reviews_dir, self.model_name, f'{self.benchmark_name}_{subset}.jsonl')
file_path = os.path.join(self.outputs.reviews_dir, f'{self.benchmark_name}_{subset}.jsonl')
# Ensure the directory exists
if self.outputs.is_make:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
@ -235,7 +235,7 @@ class CacheManager:
Returns:
Path to the reports directory for this model
"""
report_path = os.path.join(self.outputs.reports_dir, self.model_name)
report_path = self.outputs.reports_dir
# Ensure the directory exists
if self.outputs.is_make:
os.makedirs(report_path, exist_ok=True)