diff --git a/.gitignore b/.gitignore index e888d3b..b3d105c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,8 @@ env/ /outputs/ /output/ /output* - +/docker_images +/results /tau2-bench /log /docker diff --git a/bash/collect_results.py b/bash/collect_results.py new file mode 100644 index 0000000..f25cc57 --- /dev/null +++ b/bash/collect_results.py @@ -0,0 +1,631 @@ +#!/usr/bin/env python3 +""" +Collect benchmark results from an EvalScope output directory and write a summary +Excel/CSV similar to: + + P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv + +Rules: +- Scores are averaged across seeds / multi-runs. +- Perf metrics (latency, TTFT, TPOT, TPS, tokens) are recomputed from raw + predictions across all seeds / multi-runs, so resuming from a checkpoint + no longer resets cumulative statistics. +""" + +import argparse +import json +import statistics +from pathlib import Path + +import numpy as np +import pandas as pd + + +# Map benchmark -> capability domain (same as the reference CSV) +BENCHMARK_DOMAIN = { + 'bigcodebench': '代码与工程', + 'humaneval': '代码与工程', + 'live_code_bench': '代码与工程', + 'aime24': '推理与数学', + 'aime25': '推理与数学', + 'aime26': '推理与数学', + 'hmmt26': '推理与数学', + 'imo_answerbench': '推理与数学', + 'gsm8k': '推理与数学', + 'competition_math': '推理与数学', + 'bbh': '推理与数学', + 'drop': '推理与数学', + 'gpqa_diamond': '知识与语言理解', + 'hle': '知识与语言理解', + 'hle': '知识与语言理解', + 'hle_low': '知识与语言理解', + 'mmlu_pro': '知识与语言理解', + 'simple_qa': '知识与语言理解', + 'super_gpqa': '知识与语言理解', + 'mmlu': '知识与语言理解', + 'cmmlu': '知识与语言理解', + 'arc': '知识与语言理解', + 'hellaswag': '知识与语言理解', + 'trivia_qa': '知识与语言理解', + 'winogrande': '知识与语言理解', + 'longbench_v2': '长上下文', + 'openai_mrcr': '长上下文', + 'tau2_bench': '智能体与工具', + 'general_fc': '智能体与工具', + 'bfcl_v3': '智能体与工具', +} + +# Column order matching the reference CSV +OUTPUT_COLUMNS = [ + '分类', + 'Benchmark', + '得分', + '实测时间(h)', + '总样本数', + '延迟_mean(s)', + '输出TPS', + '请求QPS', + '输入tokens_mean', + '输出tokens_mean', + '累计总tokens', + 'TTFT_mean(s)', + 'TTFT P90', + 'TTFT P99', + 'TPOT_mean(s)', + 'TPOT P90', + 'TPOT P99', +] + +# Some evalscope output directory names do not match the canonical benchmark +# name (e.g. the hle benchmark writes to the `hle_low` directory because of +# subset naming). Map them to the canonical name here so the summary uses a +# consistent label. +BENCHMARK_NAME_ALIAS = { + 'hle_low': 'hle', +} + + +def percentile(values, q): + """Return the q-th percentile using numpy's linear interpolation.""" + if not values: + return np.nan + return float(np.percentile(values, q)) + + +def read_predictions(pred_file: Path): + """Yield perf_metrics dicts from a predictions JSONL file.""" + for obj in read_predictions_with_index(pred_file): + if obj['perf_metrics'] is not None: + yield obj['perf_metrics'] + + +def read_predictions_with_index(pred_file: Path): + """Yield ``{'index', 'perf_metrics'}`` dicts from a predictions JSONL file.""" + if not pred_file.exists(): + return + with open(pred_file, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + model_output = obj.get('model_output', {}) + pm = model_output.get('perf_metrics') + if not pm and 'choices' in model_output: + choices = model_output['choices'] + if choices and 'message' in choices[0]: + pm = choices[0]['message'].get('perf_metrics') + yield {'index': obj.get('index'), 'perf_metrics': pm} + + +def load_backup_summary(output_dir: Path, benchmark: str, model_name: str): + """Load the durable ``perf_stats_backup/__.json``. + + Returns the ``summary`` dict (with ``latency`` / ``ttft`` / ``tpot`` / + ``throughput`` / ``usage`` sub-dicts) or ``None`` if no backup exists. + """ + try: + from perf_backup import get_backup_paths + except ImportError: + return None + perf_path, _ = get_backup_paths(Path(output_dir), benchmark, model_name) + if not perf_path.exists(): + return None + try: + payload = json.loads(perf_path.read_text(encoding='utf-8')) + return payload.get('summary') + except Exception: + return None + + +def find_report(output_dir: Path, benchmark: str, model_name: str): + """Find the first report JSON for a benchmark/model under output_dir.""" + bench_dir = output_dir / benchmark + if not bench_dir.exists(): + return None + for seed_dir in sorted(bench_dir.iterdir()): + if not seed_dir.is_dir(): + continue + for candidate in find_report_candidates(seed_dir, benchmark, model_name): + if candidate.exists(): + return candidate + return None + + +def find_report_candidates(seed_dir: Path, benchmark: str, model_name: str): + """Return candidate report paths, accepting variant naming patterns.""" + # The actual report filename may not match the directory name exactly + # (e.g. hle_low directory holds hle.json). Try all .json files under + # the model directory. + candidates = [ + seed_dir / 'reports' / model_name / f'{benchmark}.json', + seed_dir / 'reports' / f'{benchmark}.json', + ] + for p in (seed_dir / 'reports' / model_name).glob('*.json'): + candidates.append(p) + for p in (seed_dir / 'reports').glob('*.json'): + candidates.append(p) + return candidates + + +def find_all_reports(output_dir: Path, benchmark: str, model_name: str): + """Find all report JSONs for a benchmark/model (across seeds/runs).""" + bench_dir = output_dir / benchmark + reports = [] + if not bench_dir.exists(): + return reports + for seed_dir in sorted(bench_dir.iterdir()): + if not seed_dir.is_dir(): + continue + for candidate in find_report_candidates(seed_dir, benchmark, model_name): + if candidate.exists() and candidate not in reports: + reports.append(candidate) + break + return reports + + +def find_archive_predictions(output_dir: Path, benchmark: str, model_name: str): + """Return the durable per-sample archive maintained by ``perf_backup.py``. + + The archive contains every sample ever produced across all runs and + breakpoints for this benchmark/model (deduplicated by ``index``). When + present, the summary aggregator should prefer it over the raw + ``predictions/*.jsonl`` files because the latter can be overwritten on + restart. + """ + try: + from perf_backup import get_backup_paths + except ImportError: + return [] + _, archive_path = get_backup_paths(output_dir, benchmark, model_name) + if archive_path.exists(): + return [archive_path] + return [] + + +def find_all_predictions(output_dir: Path, benchmark: str, model_name: str): + """Find all predictions JSONL files for a benchmark/model. + + The durable ``predictions_archive/__.jsonl`` (if + present) is returned **first** so its deduplicated per-sample records + dominate any smaller predictions that a fresh run may have written. + """ + files = list(find_archive_predictions(output_dir, benchmark, model_name)) + bench_dir = output_dir / benchmark + if not bench_dir.exists(): + return files + for seed_dir in sorted(bench_dir.iterdir()): + if not seed_dir.is_dir(): + continue + pred_dir = seed_dir / 'predictions' / model_name + if pred_dir.exists(): + files.extend(sorted(pred_dir.rglob('*.jsonl'))) + return files + + +def parse_log_duration(log_file: Path): + """Parse first and last timestamp from eval_log.log and return duration in hours.""" + if not log_file.exists(): + return np.nan + from datetime import datetime + first_dt = None + last_dt = None + fmt = '%Y-%m-%d %H:%M:%S' + with open(log_file, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if len(line) < 19: + continue + try: + dt = datetime.strptime(line[:19], fmt) + except ValueError: + continue + if first_dt is None: + first_dt = dt + last_dt = dt + if first_dt is None or last_dt is None or last_dt <= first_dt: + return np.nan + return (last_dt - first_dt).total_seconds() / 3600.0 + + +def extract_score(report_data: dict) -> float: + """Extract the top-level score from a report JSON.""" + score = report_data.get('score') + if score is not None: + return float(score) + metrics = report_data.get('metrics', []) + for m in metrics: + if m.get('name') == 'mean_acc': + return float(m.get('score', m.get('macro_score', 0.0))) + return 0.0 + + +def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): + """Collect aggregated results for one benchmark.""" + reports = find_all_reports(output_dir, benchmark, model_name) + if not reports: + return None + + scores = [] + summary0 = None + n_samples_unique = 0 + for report in reports: + try: + data = json.loads(report.read_text(encoding='utf-8')) + scores.append(extract_score(data)) + if summary0 is None: + summary0 = data.get('perf_metrics', {}).get('summary', {}) + n_samples_unique = summary0.get('n_samples', 0) + except Exception: + continue + + avg_score = float(np.mean(scores)) if scores else 0.0 + + # Aggregate raw prediction perf metrics across all seeds/runs. + # This fixes the breakpoint-resume issue where cumulative stats are reset. + # Predictions are deduplicated by sample `index` so the archive (which + # spans every run) and the latest ``predictions/*.jsonl`` don't double + # count the same sample. + pred_files = find_all_predictions(output_dir, benchmark, model_name) + seen_indexes = set() + latencies = [] + ttfts = [] + tpots = [] + input_tokens = [] + output_tokens = [] + sample_indexes = [] + for pf in pred_files: + for obj in read_predictions_with_index(pf): + idx = obj['index'] + if idx is None or idx in seen_indexes: + # Still keep perf data even when we lack an index, so older + # benchmark files without `index` don't get dropped. + pm = obj['perf_metrics'] + else: + seen_indexes.add(idx) + sample_indexes.append(idx) + pm = obj['perf_metrics'] + if pm is None: + continue + if 'latency' in pm: + latencies.append(float(pm['latency'])) + if 'ttft' in pm: + ttfts.append(float(pm['ttft'])) + if 'tpot' in pm: + tpots.append(float(pm['tpot'])) + + itok = pm.get('input_tokens') + otok = pm.get('output_tokens') + if itok is None and 'usage' in pm: + itok = pm['usage'].get('input_tokens') + otok = pm['usage'].get('output_tokens') + if itok is not None: + input_tokens.append(int(itok)) + if otok is not None: + output_tokens.append(int(otok)) + + # If we don't have raw predictions but have a perf_stats backup, that + # represents a known-good summary captured right after a clean run — + # preferable to summary0 (which may be the just-reset run). + backup_summary = None + if not latencies: + backup_summary = load_backup_summary(output_dir, benchmark, model_name) + + if latencies: + latency_mean = float(np.mean(latencies)) + total_compute_time = float(np.sum(latencies)) + total_output_tokens = sum(output_tokens) + avg_output_tps = total_output_tokens / total_compute_time if total_compute_time > 0 else np.nan + avg_req_ps = len(latencies) / total_compute_time if total_compute_time > 0 else np.nan + input_tok_mean = float(np.mean(input_tokens)) if input_tokens else np.nan + output_tok_mean = float(np.mean(output_tokens)) if output_tokens else np.nan + total_tokens = sum(input_tokens) + sum(output_tokens) + ttft_mean = float(np.mean(ttfts)) if ttfts else np.nan + ttft_p90 = percentile(ttfts, 90) + ttft_p99 = percentile(ttfts, 99) + tpot_mean = float(np.mean(tpots)) if tpots else np.nan + tpot_p90 = percentile(tpots, 90) + tpot_p99 = percentile(tpots, 99) + # The actual sample count we just rebuilt from raw predictions is more + # reliable than the (possibly reset) report summary's n_samples. + if sample_indexes: + n_samples_unique = len(sample_indexes) + elif summary0: + # Fallback to report summary if raw predictions are unavailable + latency_mean = summary0.get('latency', {}).get('mean', np.nan) + avg_output_tps = summary0.get('throughput', {}).get('avg_output_tps', np.nan) + avg_req_ps = summary0.get('throughput', {}).get('avg_req_ps', np.nan) + input_tok_mean = summary0.get('tokens', {}).get('input_tokens', {}).get('mean', np.nan) + output_tok_mean = summary0.get('tokens', {}).get('output_tokens', {}).get('mean', np.nan) + total_tokens = summary0.get('tokens', {}).get('total_tokens_count', np.nan) + ttft_mean = summary0.get('ttft', {}).get('mean', np.nan) + ttft_p90 = summary0.get('ttft', {}).get('90%', np.nan) + ttft_p99 = summary0.get('ttft', {}).get('99%', np.nan) + tpot_mean = summary0.get('tpot', {}).get('mean', np.nan) + tpot_p90 = summary0.get('tpot', {}).get('90%', np.nan) + tpot_p99 = summary0.get('tpot', {}).get('99%', np.nan) + # If the just-read report looks like a freshly-reset run (smaller + # n_samples than the durable backup), prefer the backup's summary so + # the cumulative numbers are not lost. + try: + from perf_backup import get_backup_paths + backup_path, _ = get_backup_paths(output_dir, benchmark, model_name) + if backup_path.exists(): + payload = json.loads(backup_path.read_text(encoding='utf-8')) + if (payload.get('n_samples') or 0) > (summary0.get('n_samples') or 0): + backup_summary = payload.get('summary') or {} + latency_mean = backup_summary.get('latency', {}).get('mean', latency_mean) + avg_output_tps = backup_summary.get('throughput', {}).get('avg_output_tps', avg_output_tps) + avg_req_ps = backup_summary.get('throughput', {}).get('avg_req_ps', avg_req_ps) + input_tok_mean = backup_summary.get('usage', {}).get('input_tokens', {}).get('mean', input_tok_mean) + output_tok_mean = backup_summary.get('usage', {}).get('output_tokens', {}).get('mean', output_tok_mean) + total_tokens = backup_summary.get('usage', {}).get('total_tokens_count', total_tokens) + ttft_mean = backup_summary.get('ttft', {}).get('mean', ttft_mean) + ttft_p90 = backup_summary.get('ttft', {}).get('90%', ttft_p90) + ttft_p99 = backup_summary.get('ttft', {}).get('99%', ttft_p99) + tpot_mean = backup_summary.get('tpot', {}).get('mean', tpot_mean) + tpot_p90 = backup_summary.get('tpot', {}).get('90%', tpot_p90) + tpot_p99 = backup_summary.get('tpot', {}).get('99%', tpot_p99) + n_samples_unique = payload.get('n_samples') or n_samples_unique + except Exception: + pass + elif backup_summary: + latency_mean = backup_summary.get('latency', {}).get('mean', np.nan) + avg_output_tps = backup_summary.get('throughput', {}).get('avg_output_tps', np.nan) + avg_req_ps = backup_summary.get('throughput', {}).get('avg_req_ps', np.nan) + input_tok_mean = backup_summary.get('usage', {}).get('input_tokens', {}).get('mean', np.nan) + output_tok_mean = backup_summary.get('usage', {}).get('output_tokens', {}).get('mean', np.nan) + total_tokens = backup_summary.get('usage', {}).get('total_tokens_count', np.nan) + ttft_mean = backup_summary.get('ttft', {}).get('mean', np.nan) + ttft_p90 = backup_summary.get('ttft', {}).get('90%', np.nan) + ttft_p99 = backup_summary.get('ttft', {}).get('99%', np.nan) + tpot_mean = backup_summary.get('tpot', {}).get('mean', np.nan) + tpot_p90 = backup_summary.get('tpot', {}).get('90%', np.nan) + tpot_p99 = backup_summary.get('tpot', {}).get('99%', np.nan) + else: + return None + + # Duration: prefer the durable active timer maintained by perf_backup.py, + # which only counts time when run_task() is actually executing. This avoids + # counting idle gaps caused by manual interruption/resume. + # If no active timer exists, fall back to log duration or latency estimate. + active_time_hours = np.nan + try: + from perf_backup import load_active_time + active_seconds = load_active_time(output_dir, benchmark, model_name) + if active_seconds > 0: + active_time_hours = active_seconds / 3600.0 + except Exception: + pass + + if not np.isnan(active_time_hours): + duration_hours = active_time_hours + else: + # Legacy fallback: sum of wall-clock durations from eval_log.log. + # For sandbox benchmarks the main log may not cover sandbox execution, + # so we also estimate wall time as compute_time/batch_size and take max. + duration_hours = 0.0 + batch_size = 4 + for report in reports: + log_file = report.parent.parent.parent / 'logs' / 'eval_log.log' + d = parse_log_duration(log_file) + if not np.isnan(d): + duration_hours += d + cfg_file = report.parent.parent.parent / 'configs' / 'task_config.yaml' + if cfg_file.exists(): + try: + import yaml + cfg = yaml.safe_load(cfg_file.read_text(encoding='utf-8')) + batch_size = int(cfg.get('eval_batch_size', batch_size)) + except Exception: + pass + + compute_wall_estimate = (total_compute_time / batch_size / 3600.0) if latencies and batch_size > 0 else np.nan + if duration_hours <= 0: + duration_hours = compute_wall_estimate + else: + duration_hours = max(duration_hours, compute_wall_estimate) + + return { + '分类': BENCHMARK_DOMAIN.get(benchmark, '其他'), + 'Benchmark': BENCHMARK_NAME_ALIAS.get(benchmark, benchmark), + '得分': round(avg_score, 4), + '实测时间(h)': round(duration_hours, 4) if not np.isnan(duration_hours) else np.nan, + '总样本数': n_samples_unique, + '延迟_mean(s)': round(latency_mean, 5) if not np.isnan(latency_mean) else np.nan, + '输出TPS': round(avg_output_tps, 2) if not np.isnan(avg_output_tps) else np.nan, + '请求QPS': round(avg_req_ps, 4) if not np.isnan(avg_req_ps) else np.nan, + '输入tokens_mean': round(input_tok_mean, 2) if not np.isnan(input_tok_mean) else np.nan, + '输出tokens_mean': round(output_tok_mean, 2) if not np.isnan(output_tok_mean) else np.nan, + '累计总tokens': total_tokens if not np.isnan(total_tokens) else np.nan, + 'TTFT_mean(s)': round(ttft_mean, 5) if not np.isnan(ttft_mean) else np.nan, + 'TTFT P90': round(ttft_p90, 5) if not np.isnan(ttft_p90) else np.nan, + 'TTFT P99': round(ttft_p99, 5) if not np.isnan(ttft_p99) else np.nan, + 'TPOT_mean(s)': round(tpot_mean, 5) if not np.isnan(tpot_mean) else np.nan, + 'TPOT P90': round(tpot_p90, 5) if not np.isnan(tpot_p90) else np.nan, + 'TPOT P99': round(tpot_p99, 5) if not np.isnan(tpot_p99) else np.nan, + } + + +def eval_benchmark(benchmark_names: list, output_dir: Path, model_name: str, + out_name: str = None, excel_output_dir: Path = None): + """Collect results for a specific list of benchmarks. + + This is the public entry point intended for use by ``run.py`` and other + scripts that already know which benchmarks they ran. + + Args: + benchmark_names: List of canonical benchmark names to aggregate (e.g. + ``['aime24', 'gsm8k', 'arc']``). Only benchmarks with reports on + disk will appear in the summary. + output_dir: EvalScope output root directory. + model_name: Model name to look up reports/predictions for. + out_name: Output file name (without extension). Defaults to the safe + model name. + excel_output_dir: Optional separate directory for the Excel summary. + If provided, the Excel file is written here while the CSV stays + under ``output_dir/results//``. + + Returns: + (csv_path, xlsx_path) tuple. + """ + return collect_all(output_dir, model_name, out_name, + include_benchmarks=benchmark_names, + excel_output_dir=excel_output_dir) + + +def collect_all(output_dir: Path, model_name: str, out_name: str = None, + include_benchmarks: list = None, + excel_output_dir: Path = None): + """Collect benchmarks under ``output_dir`` and write summary Excel/CSV.""" + if not output_dir.exists(): + raise FileNotFoundError(f'Output directory not found: {output_dir}') + + safe_model = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_') + + # Discover benchmarks + if include_benchmarks is not None: + # Whitelist mode: only aggregate benchmarks that were actually run in + # this invocation. Directory names may differ from canonical names + # (e.g. evalscope writes the hle benchmark to the hle_low directory), + # so we canonicalize each directory name via BENCHMARK_NAME_ALIAS and + # check membership in the caller's list. Order follows the caller's + # include_benchmarks list. + include_set = set(include_benchmarks) + discovered = [] # list of (dir_name, canonical_name) + for bench_dir in output_dir.iterdir(): + if not bench_dir.is_dir() or bench_dir.name == safe_model: + continue + if not find_all_reports(output_dir, bench_dir.name, model_name): + continue + canonical = BENCHMARK_NAME_ALIAS.get(bench_dir.name, bench_dir.name) + if canonical in include_set or bench_dir.name in include_set: + discovered.append((bench_dir.name, canonical)) + + order = {name: idx for idx, name in enumerate(include_benchmarks)} + discovered.sort(key=lambda x: order.get(x[1], len(include_benchmarks))) + + benchmarks = [dir_name for dir_name, _ in discovered] + else: + # Legacy behaviour: scan everything on disk. + benchmarks = [] + for bench_dir in sorted(output_dir.iterdir()): + if not bench_dir.is_dir() or bench_dir.name == safe_model: + continue + reports = find_all_reports(output_dir, bench_dir.name, model_name) + if reports: + benchmarks.append(bench_dir.name) + + rows = [] + for benchmark in benchmarks: + row = collect_benchmark(output_dir, benchmark, model_name) + if row: + rows.append(row) + + if not rows: + print(f'No results found for model {model_name} under {output_dir}') + return None, None + + df = pd.DataFrame(rows, columns=OUTPUT_COLUMNS) + df = df.drop_duplicates(subset=['Benchmark'], keep='first') + + # Add total row + total_score = df['得分'].mean() + total_time = df['实测时间(h)'].sum() + total_row = { + '分类': '总计', + 'Benchmark': '', + '得分': round(total_score, 4), + '实测时间(h)': round(total_time, 4), + } + for col in OUTPUT_COLUMNS: + if col not in total_row: + total_row[col] = np.nan + df = pd.concat([df, pd.DataFrame([total_row], columns=OUTPUT_COLUMNS)], ignore_index=True) + + summary_dir = output_dir / 'results' / safe_model + summary_dir.mkdir(parents=True, exist_ok=True) + + if out_name is None: + out_name = safe_model + + csv_path = summary_dir / f'{out_name}.csv' + + # Excel can optionally be written to a separate project-level results dir + # so that the latest summary per model is easy to find independently of + # the per-run output directory. + if excel_output_dir is not None: + xlsx_dir = Path(excel_output_dir) + xlsx_dir.mkdir(parents=True, exist_ok=True) + xlsx_path = xlsx_dir / f'{out_name}.xlsx' + else: + xlsx_path = summary_dir / f'{out_name}.xlsx' + + df.to_csv(csv_path, index=False, encoding='utf-8-sig') + df.to_excel(xlsx_path, index=False) + + print(f'Summary written to:') + print(f' CSV: {csv_path}') + print(f' Excel: {xlsx_path}') + return csv_path, xlsx_path + + +def main(): + parser = argparse.ArgumentParser(description='Collect EvalScope benchmark results into Excel/CSV') + parser.add_argument('--output-dir', required=True, help='EvalScope output directory') + parser.add_argument('--model', default='DeepSeek-V4-Flash-Int8', help='Model name used in reports') + parser.add_argument('--out-name', default=None, help='Output file name (without extension); defaults to safe model name') + parser.add_argument('--include', default=None, + help='Comma-separated benchmark whitelist (default: all on disk)') + parser.add_argument('--benchmarks', '--benchmarks', dest='benchmarks', default=None, + help='Alias for --include') + parser.add_argument('--excel-output-dir', default=None, + help='Separate directory for the Excel summary; defaults to /results') + args = parser.parse_args() + + # Default Excel output dir to project-level results/ so the latest summary + # per model is easy to find independently of the per-run output directory. + if args.excel_output_dir is None: + excel_output_dir = Path(__file__).parent.parent / 'results' + else: + excel_output_dir = Path(args.excel_output_dir) + + include_benchmarks = None + if args.include: + include_benchmarks = [b.strip() for b in args.include.split(',') if b.strip()] + elif args.benchmarks: + include_benchmarks = [b.strip() for b in args.benchmarks.split(',') if b.strip()] + + if include_benchmarks is not None: + eval_benchmark(include_benchmarks, Path(args.output_dir), args.model, args.out_name, + excel_output_dir=excel_output_dir) + else: + collect_all(Path(args.output_dir), args.model, args.out_name, + excel_output_dir=excel_output_dir) + + +if __name__ == '__main__': + main() diff --git a/bash/perf_backup.py b/bash/perf_backup.py new file mode 100644 index 0000000..c05b31f --- /dev/null +++ b/bash/perf_backup.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +""" +Perf-stats backup helpers. + +EvalScope writes the per-benchmark cumulative perf summary into +``report.json`` under ``perf_metrics.summary`` and the raw per-sample records +into ``predictions/*.jsonl``. Both are overwritten on every run, so a +checkpoint restart resets cumulative stats and loses prior samples. + +This module maintains **two durable files** under ``/`` so the +breakpoint-resume issue can be reconstructed: + + 1. ``perf_stats_backup/__.json`` + - The latest complete ``perf_metrics.summary`` captured right after + each ``run_task()`` finishes. + 2. ``predictions_archive/__.jsonl`` + - All per-sample records ever seen, deduplicated by sample ``index``. + +``bash.collect_results`` prefers these archive files when aggregating metrics. +""" + +import json +import shutil +from datetime import datetime +from pathlib import Path + + +PERF_STATS_BACKUP_DIRNAME = 'perf_stats_backup' +PREDICTIONS_ARCHIVE_DIRNAME = 'predictions_archive' +ACTIVE_TIME_DIRNAME = 'active_time' + + +def _safe(s: str) -> str: + return s.replace('/', '_').replace('\\', '_').replace(' ', '_') + + +def get_backup_paths(output_dir: Path, benchmark: str, model_name: str): + """Return the (perf_stats_backup, predictions_archive) paths for a benchmark/model.""" + safe = _safe(model_name) + backup_dir = Path(output_dir) / PERF_STATS_BACKUP_DIRNAME + archive_dir = Path(output_dir) / PREDICTIONS_ARCHIVE_DIRNAME + perf_stats_path = backup_dir / f'{_safe(benchmark)}__{safe}.json' + predictions_path = archive_dir / f'{_safe(benchmark)}__{safe}.jsonl' + return perf_stats_path, predictions_path + + +def backup_perf_stats(output_dir: Path, benchmark: str, model_name: str, + report_json: Path) -> Path: + """Copy ``perf_metrics.summary`` from a finished report to the durable backup. + + Only writes if ``report_json`` exists and contains a ``perf_metrics.summary``. + Returns the backup path. + """ + if not report_json or not report_json.exists(): + return None + try: + data = json.loads(report_json.read_text(encoding='utf-8')) + except Exception: + return None + summary = (data.get('perf_metrics') or {}).get('summary') + if not summary: + return None + + perf_stats_path, _ = get_backup_paths(Path(output_dir), benchmark, model_name) + perf_stats_path.parent.mkdir(parents=True, exist_ok=True) + payload = { + 'benchmark': benchmark, + 'model': model_name, + 'updated_at': datetime.now().isoformat(timespec='seconds'), + 'n_samples': summary.get('n_samples'), + 'summary': summary, + } + # Don't overwrite a strictly larger backup with a smaller one — that + # would be the symptom of a reset run and we want to keep the larger. + if perf_stats_path.exists(): + try: + old = json.loads(perf_stats_path.read_text(encoding='utf-8')) + old_n = old.get('n_samples') or 0 + new_n = summary.get('n_samples') or 0 + if new_n < old_n: + # Don't clobber the larger historical backup. + return perf_stats_path + except Exception: + pass + perf_stats_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding='utf-8') + return perf_stats_path + + +def archive_predictions(output_dir: Path, benchmark: str, model_name: str, + predictions_dir: Path) -> Path: + """Append the latest predictions into the durable archive. + + Reads every ``*.jsonl`` under ``predictions_dir``, parses each line, and + appends each sample to the archive file (deduplicated by ``index``). If + the same ``index`` was already archived with a different perf payload, + the newer record wins. + + Returns the archive path. + """ + _, archive_path = get_backup_paths(Path(output_dir), benchmark, model_name) + archive_path.parent.mkdir(parents=True, exist_ok=True) + + if not predictions_dir or not predictions_dir.exists(): + return archive_path + + # Load existing archive (index -> record) + existing = {} + if archive_path.exists(): + with archive_path.open('r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + idx = obj.get('index') + if idx is not None: + existing[idx] = obj + + # Add new predictions + new_count = 0 + updated_count = 0 + for jsonl in sorted(predictions_dir.rglob('*.jsonl')): + with jsonl.open('r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + idx = obj.get('index') + if idx is None: + continue + if idx in existing: + updated_count += 1 + else: + new_count += 1 + existing[idx] = obj + + # Write back atomically + tmp_path = archive_path.with_suffix('.jsonl.tmp') + with tmp_path.open('w', encoding='utf-8') as f: + for idx in sorted(existing.keys()): + f.write(json.dumps(existing[idx], ensure_ascii=False)) + f.write('\n') + shutil.move(str(tmp_path), str(archive_path)) + + return archive_path + + +def restore_from_backup(output_dir: Path, benchmark: str, model_name: str, + target_predictions_dir: Path, target_report_json: Path): + """If both ``perf_stats_backup`` and ``predictions_archive`` exist for a + benchmark/model, materialise them into ``target_predictions_dir`` and + ``target_report_json`` before the next run starts, so evalscope resumes + from the larger historical state instead of overwriting it. + + Returns ``True`` if a backup was restored, ``False`` otherwise. + """ + perf_stats_path, archive_path = get_backup_paths(Path(output_dir), benchmark, model_name) + if not perf_stats_path.exists() and not archive_path.exists(): + return False + + # Restore the archive into the predictions directory so evalscope's + # use_cache mechanism sees the previously-completed samples. + if archive_path.exists(): + target_predictions_dir.mkdir(parents=True, exist_ok=True) + target_jsonl = target_predictions_dir / archive_path.name + shutil.copy2(archive_path, target_jsonl) + + # Restore the perf summary into the report file so the cumulative stats + # survive. We do NOT overwrite score/metrics — only the perf section. + if perf_stats_path.exists() and target_report_json.exists(): + try: + payload = json.loads(perf_stats_path.read_text(encoding='utf-8')) + summary = payload.get('summary') + if not summary: + return True + report = json.loads(target_report_json.read_text(encoding='utf-8')) + report.setdefault('perf_metrics', {})['summary'] = summary + target_report_json.write_text( + json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8' + ) + except Exception: + pass + + return True + + +# ============================================================ +# Active run-time tracking (excludes idle gaps between resumes) +# ============================================================ + + +def get_active_time_path(output_dir: Path, benchmark: str, model_name: str) -> Path: + """Return the path to the durable active-time record for a benchmark/model.""" + safe = _safe(model_name) + active_dir = Path(output_dir) / ACTIVE_TIME_DIRNAME + return active_dir / f'{_safe(benchmark)}__{safe}.json' + + +def record_active_time(output_dir: Path, benchmark: str, model_name: str, + seconds: float) -> Path: + """Accumulate ``seconds`` of active run time for a benchmark/model. + + This is intended to measure only the time ``run_task()`` was actually + executing, excluding idle gaps caused by manual interruption/resume. + """ + path = get_active_time_path(Path(output_dir), benchmark, model_name) + path.parent.mkdir(parents=True, exist_ok=True) + + data = {'total_seconds': 0.0} + if path.exists(): + try: + data = json.loads(path.read_text(encoding='utf-8')) + except Exception: + pass + + data['total_seconds'] = float(data.get('total_seconds', 0.0)) + float(seconds) + data['updated_at'] = datetime.now().isoformat(timespec='seconds') + data['benchmark'] = benchmark + data['model'] = model_name + + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') + return path + + +def load_active_time(output_dir: Path, benchmark: str, model_name: str) -> float: + """Return accumulated active run time in seconds, or 0.0 if no record.""" + path = get_active_time_path(Path(output_dir), benchmark, model_name) + if not path.exists(): + return 0.0 + try: + data = json.loads(path.read_text(encoding='utf-8')) + return float(data.get('total_seconds', 0.0)) + except Exception: + return 0.0 diff --git a/bash/pull_swe_bench_images.py b/bash/pull_swe_bench_images.py new file mode 100755 index 0000000..c60048c --- /dev/null +++ b/bash/pull_swe_bench_images.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +""" +预拉取 SWE-bench 评测所需的 Docker 镜像，支持断点续传。 + +用法: + # 默认从上次进度继续 + python bash/pull_swe_bench_images.py --dataset swe_bench_verified --max-workers 4 + + # 强制重新开始（删除进度文件） + rm -f output/.swe_bench_pull_state.json + python bash/pull_swe_bench_images.py --dataset swe_bench_verified --max-workers 4 + + # 测试前 3 个镜像 + python bash/pull_swe_bench_images.py --dry-run 3 --max-workers 1 + +进度文件: + output/.swe_bench_pull_state.json + 记录已拉取/跳过/失败的镜像，Ctrl+C 后下次自动跳过。 +""" + +import json +import os +import sys +import threading +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +# 国内网络加速（必须在导入 evalscope/huggingface 之前设置） +os.environ.setdefault('USE_MODELSCOPE_HUB', '1') +os.environ.setdefault('HF_ENDPOINT', 'https://hf-mirror.com') +os.environ.setdefault('PYTHONUNBUFFERED', '1') + +from tqdm import tqdm + +# 让 evalscope 可导入 +PROJECT_ROOT = Path(__file__).parent.parent.resolve() +sys.path.insert(0, str(PROJECT_ROOT / 'evalscope')) + +from evalscope.api.dataset import RemoteDataLoader, FieldSpec + + +DATASET_IDS = { + 'swe_bench_verified': 'princeton-nlp/SWE-bench_Verified', + # 'swe_bench_lite': 'princeton-nlp/SWE-bench_Lite', + # 'swe_bench_verified_mini': 'evalscope/swe-bench-verified-mini', + # 'swe_bench_pro': 'evalscope/swe-bench-pro', +} + +DEFAULT_OUTPUT_DIR = PROJECT_ROOT / 'output' +STATE_FILENAME = '.swe_bench_pull_state.json' + + +def get_state_path(output_dir: Path) -> Path: + return Path(output_dir) / STATE_FILENAME + + +def load_state(output_dir: Path) -> dict: + """Load previous pull state; return empty state if none exists.""" + state_path = get_state_path(output_dir) + if not state_path.exists(): + return {'done': [], 'skipped': [], 'failed': []} + try: + with state_path.open('r', encoding='utf-8') as f: + state = json.load(f) + # Ensure required keys exist + for key in ('done', 'skipped', 'failed'): + state.setdefault(key, []) + return state + except Exception as e: + print(f'WARNING: failed to load state file {state_path}: {e}') + return {'done': [], 'skipped': [], 'failed': []} + + +def save_state(output_dir: Path, state: dict): + """Atomically save current pull state.""" + state_path = get_state_path(output_dir) + state_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = state_path.with_suffix('.tmp') + try: + with tmp_path.open('w', encoding='utf-8') as f: + json.dump(state, f, ensure_ascii=False, indent=2) + tmp_path.replace(state_path) + except Exception as e: + print(f'WARNING: failed to save state file {state_path}: {e}') + + +def load_swe_samples(dataset_id: str): + """Load SWE-bench samples and wrap them into the format expected by build_images.""" + print(f'正在加载数据集元数据: {dataset_id} ...') + loader = RemoteDataLoader( + data_id_or_path=dataset_id, + split='test', + sample_fields=FieldSpec( + input='problem_statement', + metadata=[ + 'instance_id', + 'repo', + 'base_commit', + 'patch', + 'PASS_TO_PASS', + 'FAIL_TO_PASS', + 'test_patch', + 'version', + 'environment_setup_commit', + 'hints_text', + 'created_at', + ], + ), + ) + return loader.load() + + +def get_image_name(instance, namespace: str, arch: str) -> str: + """根据 swebench 规则生成远程镜像名称。""" + if isinstance(instance, dict): + metadata = instance + else: + metadata = getattr(instance, 'metadata', {}) or {} + + instance_id = metadata.get('instance_id', '') + if not instance_id and not isinstance(instance, dict): + instance_id = getattr(instance, 'instance_id', '') + + if not instance_id: + return None + + image_name = f'{namespace}/sweb.eval.{arch}.{instance_id.lower()}:latest' + image_name = image_name.replace('__', '_1776_') + return image_name + + +def get_local_images() -> set: + """Return a set of all locally available Docker image names.""" + result = subprocess.run( + ['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'], + capture_output=True, text=True + ) + if result.returncode != 0: + return set() + return {line.strip() for line in result.stdout.splitlines() if line.strip()} + + +_print_lock = threading.Lock() + + +def _log(msg: str): + with _print_lock: + print(msg, flush=True) + + +def _stream_pull(image_name: str) -> tuple: + """Run docker pull and stream output in real time. + + Returns (returncode, last_lines_of_output). + """ + process = subprocess.Popen( + ['docker', 'pull', image_name], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + lines = [] + if process.stdout: + for line in process.stdout: + line = line.rstrip('\n') + lines.append(line) + _log(f' {line}') + # Keep only the last 50 lines for error reporting + if len(lines) > 50: + lines.pop(0) + + process.wait() + return process.returncode, '\n'.join(lines) + + +def pull_one_image(args_tuple, state: dict, output_dir: Path, local_images: set): + """Pull a single image and update shared state. + + Returns (idx, image_name, status, message). + """ + idx, total, instance, namespace, arch = args_tuple + image_name = get_image_name(instance, namespace, arch) + if not image_name: + return idx, None, 'skip', 'missing instance_id' + + # Resume: skip if already recorded in state + if image_name in state['done'] or image_name in state['skipped']: + return idx, image_name, 'skip', 'already recorded in state' + + # Also check local docker images (in case state was deleted) + if image_name in local_images: + state['skipped'].append(image_name) + save_state(output_dir, state) + return idx, image_name, 'skip', 'already exists locally' + + _log(f'⬇️ [{idx+1}/{total}] START {image_name}') + + rc, output = _stream_pull(image_name) + + if rc == 0: + state['done'].append(image_name) + save_state(output_dir, state) + return idx, image_name, 'success', '' + else: + msg = output.strip()[-500:] if output.strip() else 'docker pull failed' + state['failed'].append(image_name) + save_state(output_dir, state) + return idx, image_name, 'fail', msg + + +def main(): + import argparse + parser = argparse.ArgumentParser(description='Pre-pull SWE-bench Docker images with resume support') + parser.add_argument( + '--dataset', + default='swe_bench_verified', + choices=list(DATASET_IDS.keys()), + help='SWE-bench dataset variant', + ) + parser.add_argument( + '--max-workers', + type=int, + default=4, + help='Max concurrent docker pulls (default: 4)', + ) + parser.add_argument( + '--force-arch', + default='x86_64', + choices=['', 'arm64', 'x86_64'], + help='Image architecture (default: x86_64)', + ) + parser.add_argument( + '--dockerhub-username', + default='swebench', + help='DockerHub namespace for remote images (default: swebench)', + ) + parser.add_argument( + '--output-dir', + default=str(DEFAULT_OUTPUT_DIR), + help='Directory to store progress state (default: output/)', + ) + parser.add_argument( + '--dry-run', + type=int, + default=0, + help='Only test the first N images', + ) + parser.add_argument( + '--retry-failed', + action='store_true', + help='Retry images that failed in previous runs', + ) + args = parser.parse_args() + + output_dir = Path(args.output_dir) + state = load_state(output_dir) + + # Optionally retry previously failed images + if args.retry_failed: + failed = state['failed'] + state['failed'] = [] + print(f'🔄 重试 {len(failed)} 个之前失败的镜像') + + dataset_id = DATASET_IDS[args.dataset] + samples = load_swe_samples(dataset_id) + + if args.dry_run > 0: + samples = samples[:args.dry_run] + print(f'⚠️ 测试模式：仅处理前 {args.dry_run} 个样本以验证镜像名和网络...') + else: + print(f'✅ 成功加载 {len(samples)} 个样本。') + + already_done = len(state['done']) + len(state['skipped']) + print(f'📂 进度文件: {get_state_path(output_dir)}') + print(f' 已下载/已存在: {len(state["done"])} | 已跳过: {len(state["skipped"])} | 历史失败: {len(state["failed"])}') + + # Cache local image names once to avoid 500+ `docker images -q` calls. + print('🔄 正在扫描本地镜像...') + local_images = get_local_images() + print(f' 本地已有 {len(local_images)} 个镜像') + print('-' * 60) + + tasks = [ + (idx, len(samples), sample, args.dockerhub_username, args.force_arch) + for idx, sample in enumerate(samples) + ] + + success_count = 0 + skip_count = 0 + fail_count = 0 + + with ThreadPoolExecutor(max_workers=args.max_workers) as executor: + futures = {executor.submit(pull_one_image, t, state, output_dir, local_images): t[0] for t in tasks} + for future in tqdm(as_completed(futures), total=len(futures), desc='Pulling images', unit='img', mininterval=1.0): + idx = futures[future] + try: + _, image_name, status, msg = future.result() + except Exception as e: + image_name, status, msg = None, 'fail', str(e) + + if status == 'success': + success_count += 1 + _log(f'✅ [{idx+1}/{len(samples)}] DONE {image_name}') + elif status == 'skip': + skip_count += 1 + _log(f'⏭️ [{idx+1}/{len(samples)}] SKIP {image_name or msg}') + else: + fail_count += 1 + _log(f'❌ [{idx+1}/{len(samples)}] FAILED {image_name}') + if msg: + _log(f' {msg}') + + print('=' * 60) + print('🎉 预拉取任务结束！') + print(f' 总计: {len(samples)} | 已存在/跳过: {skip_count} | 本次成功: {success_count} | 本次失败: {fail_count}') + print(f' 进度文件: {get_state_path(output_dir)}') + print('=' * 60) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + print('\n\n⚠️ 用户手动中断。进度已保存，下次运行会自动跳过已完成的镜像。') + sys.exit(0) + diff --git a/bash/run.py b/bash/run.py index 850e6a0..2a51d09 100644 --- a/bash/run.py +++ b/bash/run.py @@ -31,6 +31,7 @@ Examples: import argparse import json import sys +import time from copy import deepcopy from pathlib import Path @@ -43,6 +44,11 @@ from evalscope.config import SandboxTaskConfig SCRIPT_DIR = Path(__file__).parent.resolve() PROJECT_ROOT = SCRIPT_DIR.parent +# Make collect_results importable +sys.path.insert(0, str(SCRIPT_DIR)) +import collect_results as collect_results_module +import perf_backup as perf_backup_module + # ============================================================ # Default configuration (override via CLI) # ============================================================ @@ -231,6 +237,10 @@ def build_parser(): parser.add_argument('--truncation-tokens', type=int, default=DEFAULT_TRUNCATION_TOKENS, help='Middle-truncation token budget for long-context benchmarks (default: %(default)s)') + # Result collection + parser.add_argument('--no-summary', dest='write_summary', action='store_false', + help='Skip writing summary Excel/CSV after each benchmark') + return parser @@ -433,6 +443,95 @@ def build_task_config( # Main # ============================================================ +def write_summary(output_dir: str, model_name: str, + benchmark_names: list = None): + """Re-aggregate results for the given benchmarks (or all on disk if None).""" + try: + # Derive a safe file name from the model name + safe_name = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_') + excel_output_dir = PROJECT_ROOT / 'results' + if benchmark_names is not None: + collect_results_module.eval_benchmark( + benchmark_names, Path(output_dir), model_name, safe_name, + excel_output_dir=excel_output_dir, + ) + else: + collect_results_module.collect_all( + Path(output_dir), model_name, safe_name, + excel_output_dir=excel_output_dir, + ) + except Exception as e: + print(f'WARNING: failed to write summary: {e}') + + +def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, + model_name: str, benchmark_names: list): + """Run a benchmark task and optionally refresh the summary table. + + ``benchmark_names`` is the running list of canonical benchmark names that + were scheduled for this invocation. The summary table will be limited to + these benchmarks so it does not pick up older results left in the output + directory from previous runs. + + After a successful ``run_task()`` we also snapshot the cumulative + ``perf_metrics.summary`` and the per-sample predictions into the durable + backup files maintained by ``perf_backup.py`` so checkpoint restarts can + recover them. + """ + dataset_name = task_cfg.datasets[0] + work_dir = Path(task_cfg.work_dir) + restore_before_run(output_dir, dataset_name, model_name, work_dir) + + start_ts = time.monotonic() + try: + run_task(task_cfg) + finally: + elapsed = time.monotonic() - start_ts + perf_backup_module.record_active_time(output_dir, dataset_name, model_name, elapsed) + print(f'Active time for {dataset_name}: {elapsed:.1f}s (total accumulated)') + + backup_after_run(output_dir, dataset_name, model_name, work_dir) + if write_summary_flag: + write_summary(output_dir, model_name, benchmark_names=benchmark_names) + + +def backup_after_run(output_dir: str, benchmark: str, model_name: str, + work_dir: Path): + """Snapshot the just-finished run's perf summary and predictions.""" + try: + report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + perf_backup_module.backup_perf_stats( + Path(output_dir), benchmark, model_name, report_json, + ) + predictions_dir = work_dir / 'predictions' / model_name + perf_backup_module.archive_predictions( + Path(output_dir), benchmark, model_name, predictions_dir, + ) + except Exception as e: + print(f'WARNING: perf backup for {benchmark} failed: {e}') + + +def restore_before_run(output_dir: str, benchmark: str, model_name: str, + work_dir: Path) -> bool: + """If durable backups exist for a benchmark/model, materialise them into + the new ``work_dir`` before evalscope starts so the next run resumes from + the larger historical state instead of overwriting it. + """ + try: + predictions_dir = work_dir / 'predictions' / model_name + report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + restored = perf_backup_module.restore_from_backup( + Path(output_dir), benchmark, model_name, + predictions_dir, report_json, + ) + if restored: + print(f'Restored {benchmark}/{model_name} from backup before run') + return restored + except Exception as e: + print(f'WARNING: perf restore for {benchmark} failed: {e}') + return False + + def main(): parser = build_parser() args = parser.parse_args() @@ -499,65 +598,60 @@ def main(): print(f'Multi-run datasets: {multi_run}') print(f'Single-run datasets: {single_run}') print(f'Agent datasets: {agent}') + print(f'Write summary: {args.write_summary}') print('=' * 60) + def run_one(dataset_name, run_idx=0, benchmark_names=None): + ds_cfg = dataset_configs[dataset_name] + task_cfg = build_task_config( + dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, + args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, + run_idx=run_idx, + ) + try: + run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, + benchmark_names=benchmark_names) + except Exception as e: + print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}') + + # Keep the ordered list of benchmarks actually run in this invocation so + # the summary table only includes them, not stale results from earlier runs. + benchmark_names = [] + for dataset_name in multi_run: if dataset_name not in dataset_configs: print(f'WARNING: {dataset_name} not in YAML config, skipping') continue - ds_cfg = dataset_configs[dataset_name] + benchmark_names.append(dataset_name) num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) for run_idx in range(num_runs): print(f"\n{'='*60}") print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})') print(f"{'='*60}") - task_cfg = build_task_config( - dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, - run_idx=run_idx, - ) - try: - run_task(task_cfg) - except Exception as e: - print(f'ERROR in {dataset_name} (run {run_idx + 1}): {e}') - continue + run_one(dataset_name, run_idx=run_idx, benchmark_names=benchmark_names) for dataset_name in single_run: if dataset_name not in dataset_configs: print(f'WARNING: {dataset_name} not in YAML config, skipping') continue - ds_cfg = dataset_configs[dataset_name] + benchmark_names.append(dataset_name) print(f"\n{'='*60}") print(f'Running: {dataset_name} (seed={args.seed})') print(f"{'='*60}") - task_cfg = build_task_config( - dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, - ) - try: - run_task(task_cfg) - except Exception as e: - print(f'ERROR in {dataset_name}: {e}') - continue + run_one(dataset_name, benchmark_names=benchmark_names) for dataset_name in agent: if dataset_name not in dataset_configs: print(f'WARNING: {dataset_name} not in YAML config, skipping') continue - ds_cfg = dataset_configs[dataset_name] + benchmark_names.append(dataset_name) print(f"\n{'='*60}") print(f'Running: {dataset_name} (seed={args.seed})') print(f"{'='*60}") - task_cfg = build_task_config( - dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, - args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, - ) - try: - run_task(task_cfg) - except Exception as e: - print(f'ERROR in {dataset_name}: {e}') - continue + run_one(dataset_name, benchmark_names=benchmark_names) + if args.write_summary: + write_summary(args.output_dir, args.model, benchmark_names=benchmark_names) print('\nAll benchmarks done!') diff --git a/bash/set.sh b/bash/set.sh new file mode 100644 index 0000000..ac43e2a --- /dev/null +++ b/bash/set.sh @@ -0,0 +1,9 @@ +docker run -it --rm \ + --network host \ + -v /data1/sora/evalscope:/opt/evalscope \ + -v /data1/sora/models:/opt/models \ + -v /var/run/docker.sock:/var/run/docker.sock \ + evalscope-complete-py312:latest \ + bash + + diff --git a/bash/test.py b/bash/test.py new file mode 100644 index 0000000..6c7c285 --- /dev/null +++ b/bash/test.py @@ -0,0 +1,659 @@ +#!/usr/bin/env python3 +""" +Unified benchmark runner for EvalScope. + +A single entry point for lite / mid / full / group1 / group2 / group3 evaluations. +All tunable parameters can be controlled via command-line arguments. + +Examples: + # Full evaluation (all benchmarks, multi-run for stability) + python bash/run.py \ + --model DeepSeek-V4-Flash-Int8 \ + --api-url http://localhost:30000/v1 \ + --dataset-dir /data1/sora/evalscope \ + --output-dir /data1/sora/evalscope/output \ + --suite full \ + --limit none + + # Lite smoke test (~5h with full samples) + python bash/run.py --suite lite --limit none + + # Run only selected benchmarks + python bash/run.py --datasets aime24,gsm8k,arc --limit 20 + + # Custom judge model + python bash/run.py \ + --judge-model deepseek-v4-pro \ + --judge-api-url https://api.deepseek.com/v1 \ + --judge-api-key sk-xxx +""" + +import argparse +import json +import sys +import time +from copy import deepcopy +from pathlib import Path + +import yaml + +from evalscope import run_task, TaskConfig +from evalscope.api.agent import NativeAgentConfig +from evalscope.config import SandboxTaskConfig + +SCRIPT_DIR = Path(__file__).parent.resolve() +PROJECT_ROOT = SCRIPT_DIR.parent + +# Make collect_results importable +sys.path.insert(0, str(SCRIPT_DIR)) +import collect_results as collect_results_module +import perf_backup as perf_backup_module + +# ============================================================ +# Default configuration (override via CLI) +# ============================================================ + +DEFAULT_MODEL = 'DeepSeek-V4-Flash-Int8' +DEFAULT_API_URL = 'http://localhost:30000/v1' +DEFAULT_DATASET_DIR = str(PROJECT_ROOT) +DEFAULT_OUTPUT_DIR = str(PROJECT_ROOT / 'output') +DEFAULT_CONFIG = str(PROJECT_ROOT / 'config' / 'dpv4-int8_nothinking.yaml') +DEFAULT_TOKENIZER_PATH = '/data1/models/DeepSeek-V4-Flash-INT8' +DEFAULT_LIMIT = None +DEFAULT_SEED = 42 +DEFAULT_BATCH_SIZE = 4 +DEFAULT_ENABLE_THINKING = False + +DEFAULT_JUDGE_MODEL = 'DeepSeek/DeepSeek-V4-Pro' +DEFAULT_JUDGE_API_URL = 'https://api.vectron.meta-stone.com/v1' +DEFAULT_JUDGE_API_KEY = 'sk-dbd8a665f7634081b87ec409c7636500' +DEFAULT_JUDGE_MAX_TOKENS = 10240 + +# 长文本 middle-truncation 上限（token 数）。当前默认 128k。 +DEFAULT_TRUNCATION_TOKENS = 32768 * 4 + +# ============================================================ +# Benchmark suites +# ============================================================ + +# 多次采样配置：总样本数控制在 ~400-500 +MULTI_RUN_CONFIG = { + 'aime24': 12, + 'aime25': 12, + 'aime26': 12, + 'hmmt26': 12, + 'live_code_bench': 5, + 'imo_answerbench': 4, + 'humaneval': 3, + 'gpqa_diamond': 2, +} + +# 能力域完整列表 +ALL_MULTI_RUN = [ + # 'humaneval', 'live_code_bench', + # 'aime24', 'aime25', 'aime26', 'hmmt26', + # 'imo_answerbench', 'gpqa_diamond', +] + + +ALL_SINGLE_RUN = [ + # 'bigcodebench', 'bfcl_v3', 'competition_math', 'gsm8k', 'hle', 'super_gpqa', + # 'arc', 'bbh', 'cmmlu', 'drop', 'hellaswag', 'mmlu', 'mmlu_pro', + # 'simple_qa', 'trivia_qa', 'winogrande', + # 'openai_mrcr', 'longbench_v2', + 'swe_bench_verified', 'swe_bench_pro', +] +ALL_AGENT = [ + # 'tau2_bench', 'general_fc' + ] + +# 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡： +# Group1: ~61h | Group2: ~62h | Group3: ~55h +SUITES = { + 'full': { + 'multi': ALL_MULTI_RUN, + 'single': ALL_SINGLE_RUN, + 'agent': ALL_AGENT, + }, + 'lite': { + 'multi': ['aime24', 'humaneval'], + 'single': ['gsm8k', 'arc', 'longbench_v2'], + 'agent': ['general_fc'], + }, + 'mid': { + 'multi': ['aime24', 'humaneval'], + 'single': [ + 'live_code_bench', 'bigcodebench', 'competition_math', 'gsm8k', + 'gpqa_diamond', 'mmlu_pro', 'simple_qa', 'longbench_v2', 'openai_mrcr', + ], + 'agent': ['general_fc', 'tau2_bench'], + }, + # 多机组分组，基于 CSV 实测完整时间（已含 multi-run）平衡： + # Group1: ~22.7h | Group2: ~25.6h | Group3: ~27.0h | 合计 ~75.2h + 'group1': { + 'multi': ['live_code_bench', 'aime24', 'aime25', 'aime26', 'hmmt26', 'imo_answerbench', 'humaneval'], + 'single': ['bigcodebench', 'competition_math', 'gsm8k', 'drop', 'arc', 'hellaswag', 'winogrande'], + 'agent': [], + }, + 'group2': { + 'multi': [], + 'single': ['hle', 'mmlu_pro', 'trivia_qa'], + 'agent': [], + }, + 'group3': { + 'multi': ['gpqa_diamond'], + 'single': ['openai_mrcr', 'longbench_v2', 'bfcl_v3', 'mmlu', 'cmmlu', 'bbh', 'simple_qa'], + 'agent': ['tau2_bench', 'general_fc'], + }, +} + +# ============================================================ +# Fixed configuration +# ============================================================ + +MATH_DATASETS = { + 'aime24', 'aime25', 'aime26', 'hmmt26', + 'gsm8k', 'competition_math', 'imo_answerbench', +} + +MATH_PROMPT_TEMPLATE = ( + "{question}\n" + "Please reason step by step, and put your final answer within \\boxed{{}}." +) + +SANDBOX_DATASETS = {'humaneval', 'bigcodebench', 'swe_bench_verified', 'swe_bench_pro'} +SANDBOX_CONFIGS = { + 'bigcodebench': { + 'image': 'bigcodebench-sandbox:latest', + 'working_dir': '/tmp', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, + 'humaneval': { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, + 'swe_bench_verified': { + 'image': 'swe-bench-sandbox:latest', + 'working_dir': '/tmp', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, + 'swe_bench_pro': { + 'image': 'swe-bench-sandbox:latest', + 'working_dir': '/tmp', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, +} + + +# ============================================================ +# CLI parser +# ============================================================ + +def build_parser(): + parser = argparse.ArgumentParser( + description='Unified EvalScope benchmark runner', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog='Suites: full, lite, mid, group1, group2, group3', + ) + + # Model / API + parser.add_argument('--model', default=DEFAULT_MODEL, + help='Served model name (default: %(default)s)') + parser.add_argument('--api-url', default=DEFAULT_API_URL, + help='OpenAI-compatible API URL (default: %(default)s)') + + # Paths + parser.add_argument('--dataset-dir', default=DEFAULT_DATASET_DIR, + help='Parent directory containing datasets/ subdir (default: %(default)s)') + parser.add_argument('--output-dir', default=DEFAULT_OUTPUT_DIR, + help='Output root directory (default: %(default)s)') + parser.add_argument('--config', default=DEFAULT_CONFIG, + help='YAML config path (default: %(default)s)') + parser.add_argument('--tokenizer-path', default=DEFAULT_TOKENIZER_PATH, + help='Local tokenizer path for middle-truncation (default: %(default)s)') + + # Run control + parser.add_argument('--suite', default='full', choices=list(SUITES.keys()), + help='Benchmark suite to run (default: %(default)s)') + parser.add_argument('--datasets', '--benchmarks', dest='datasets', default=None, + help='Override suite with comma-separated benchmark names, e.g. aime24,gsm8k') + parser.add_argument('--exclude', default=None, + help='Comma-separated benchmarks to exclude from the chosen suite') + parser.add_argument('--limit', default=None, + help='Max samples per benchmark; "none"/"all" for no limit (default: none)') + parser.add_argument('--seed', type=int, default=DEFAULT_SEED, + help='Random seed (default: %(default)s)') + parser.add_argument('--batch-size', type=int, default=DEFAULT_BATCH_SIZE, + help='Evaluation batch size (default: %(default)s)') + + # Decoding / thinking + parser.add_argument('--thinking', action='store_true', default=None, + help='Enable thinking mode (sglang chat_template_kwargs.thinking=True)') + parser.add_argument('--no-thinking', dest='thinking', action='store_false', + help='Disable thinking mode (default)') + + # Judge model + parser.add_argument('--judge-model', default=DEFAULT_JUDGE_MODEL, + help='Judge model name (default: %(default)s)') + parser.add_argument('--judge-api-url', default=DEFAULT_JUDGE_API_URL, + help='Judge model API URL (default: %(default)s)') + parser.add_argument('--judge-api-key', default=DEFAULT_JUDGE_API_KEY, + help='Judge model API key') + parser.add_argument('--judge-max-tokens', type=int, default=DEFAULT_JUDGE_MAX_TOKENS, + help='Judge model max_tokens (default: %(default)s)') + + # Truncation + parser.add_argument('--truncation-tokens', type=int, default=DEFAULT_TRUNCATION_TOKENS, + help='Middle-truncation token budget for long-context benchmarks (default: %(default)s)') + + # Result collection + parser.add_argument('--no-summary', dest='write_summary', action='store_false', + help='Skip writing summary Excel/CSV after each benchmark') + parser.add_argument('--summary-name', default=None, + help='Output summary file name (without extension); defaults to safe model name') + + return parser + + +# ============================================================ +# Middle-truncation helpers +# ============================================================ + +_TOKENIZER = None + + +def get_tokenizer(tokenizer_path: str): + global _TOKENIZER + if _TOKENIZER is None: + from transformers import AutoTokenizer + try: + _TOKENIZER = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) + except Exception: + _TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True) + return _TOKENIZER + + +def truncate_middle(text: str, max_tokens: int, tokenizer_path: str) -> str: + if max_tokens <= 0: + return text + tokenizer = get_tokenizer(tokenizer_path) + token_ids = tokenizer.encode(text, add_special_tokens=False) + if len(token_ids) <= max_tokens: + return text + keep_head = max_tokens // 2 + keep_tail = max_tokens - keep_head + truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:] + return tokenizer.decode(truncated_ids, skip_special_tokens=True) + + +def _patch_adapters_for_truncation(tokenizer_path: str, truncation_tokens: int): + from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter + from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter + + _orig_longbench_format = LongBenchV2Adapter.format_prompt_template + + def _patched_longbench_format(self, sample): + if sample.metadata and 'context' in sample.metadata: + sample.metadata['context'] = truncate_middle(sample.metadata['context'], truncation_tokens, tokenizer_path) + return _orig_longbench_format(self, sample) + + LongBenchV2Adapter.format_prompt_template = _patched_longbench_format + + _orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample + + def _patched_mrcr_record(self, record): + per_msg_max_tok = 8192 + if 'prompt' in record: + try: + prompt_data = json.loads(record['prompt']) + if not isinstance(prompt_data, list) or len(prompt_data) == 0: + return _orig_mrcr_record(self, record) + + tokenizer = get_tokenizer(tokenizer_path) + total_tok = sum( + len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False)) + for msg in prompt_data + ) + if total_tok <= truncation_tokens: + return _orig_mrcr_record(self, record) + + desired_idx = record.get('desired_msg_index', 0) + if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data): + desired_idx = 0 + + n = len(prompt_data) + keep = set() + keep.update(range(min(2, n))) + keep.update(range(max(0, n - 2), n)) + window = 2 + keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1))) + keep = sorted(keep) + + new_prompt = [] + for idx in keep: + msg = prompt_data[idx] + if isinstance(msg, dict): + msg = dict(msg) + content = msg.get('content', '') + if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok: + msg['content'] = truncate_middle(content, per_msg_max_tok, tokenizer_path) + new_prompt.append(msg) + + record = dict(record) + record['prompt'] = json.dumps(new_prompt) + except (json.JSONDecodeError, TypeError): + pass + return _orig_mrcr_record(self, record) + + OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record + + +# ============================================================ +# Helpers +# ============================================================ + +def load_dataset_configs(config_path: str): + if not Path(config_path).exists(): + raise FileNotFoundError(f'Config file not found: {config_path}') + with open(config_path, 'r', encoding='utf-8') as f: + return yaml.safe_load(f) + + +def configure_thinking(generation_config: dict, enable: bool) -> dict: + extra_body = generation_config.get('extra_body', {}) + chat_template_kwargs = extra_body.get('chat_template_kwargs', {}) + if enable: + chat_template_kwargs['thinking'] = True + else: + chat_template_kwargs.pop('thinking', None) + if chat_template_kwargs: + extra_body['chat_template_kwargs'] = chat_template_kwargs + if extra_body: + generation_config['extra_body'] = extra_body + return generation_config + + +def build_agent_config(agent_cfg: dict) -> NativeAgentConfig: + agent_cfg = deepcopy(agent_cfg or {}) + known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'} + kwargs = agent_cfg.pop('kwargs', {}) + for key in list(agent_cfg.keys()): + if key not in known_fields: + kwargs[key] = agent_cfg.pop(key) + if kwargs: + agent_cfg['kwargs'] = kwargs + return NativeAgentConfig(**agent_cfg) + + +def build_task_config( + dataset_name: str, + ds_cfg: dict, + batch_size: int, + enable_thinking: bool, + seed: int, + limit, + output_dir: str, + model: str, + api_url: str, + dataset_dir: str, + judge_model_args: dict, + run_idx: int = 0, +) -> TaskConfig: + if run_idx > 0: + work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}' + else: + work_dir = Path(output_dir) / dataset_name / f'seed_{seed}' + work_dir.mkdir(parents=True, exist_ok=True) + work_dir = str(work_dir) + + generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking) + dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) + dataset_args.setdefault('shuffle', True) + + if dataset_name in MATH_DATASETS: + dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE + + dataset_args_dict = {dataset_name: dataset_args} + + agent_config = None + if 'agent_config' in ds_cfg: + agent_config = build_agent_config(ds_cfg['agent_config']) + + return TaskConfig( + model=model, + api_url=api_url, + eval_type='openai_api', + dataset_dir=dataset_dir, + judge_model_args=judge_model_args, + seed=seed, + limit=limit, + collect_perf=True, + no_timestamp=True, + work_dir=work_dir, + use_cache=work_dir, + datasets=[dataset_name], + generation_config=generation_config, + dataset_args=dataset_args_dict, + agent_config=agent_config, + eval_batch_size=batch_size, + sandbox=SandboxTaskConfig( + enabled=True, + engine='docker', + default_config=SANDBOX_CONFIGS.get(dataset_name, { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }) + ) if dataset_name in SANDBOX_DATASETS else None, + ) + + +# ============================================================ +# Main +# ============================================================ + +def write_summary(output_dir: str, model_name: str, summary_name: str): + """Re-aggregate results across all benchmarks under output_dir.""" + try: + excel_output_dir = PROJECT_ROOT / 'results' + collect_results_module.collect_all( + Path(output_dir), model_name, summary_name, + excel_output_dir=excel_output_dir, + ) + except Exception as e: + print(f'WARNING: failed to write summary: {e}') + + +def backup_after_run(output_dir: str, benchmark: str, model_name: str, + work_dir: Path): + """Snapshot the just-finished run's perf summary and predictions.""" + try: + report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + perf_backup_module.backup_perf_stats( + Path(output_dir), benchmark, model_name, report_json, + ) + predictions_dir = work_dir / 'predictions' / model_name + perf_backup_module.archive_predictions( + Path(output_dir), benchmark, model_name, predictions_dir, + ) + except Exception as e: + print(f'WARNING: perf backup for {benchmark} failed: {e}') + + +def restore_before_run(output_dir: str, benchmark: str, model_name: str, + work_dir: Path) -> bool: + """If durable backups exist for a benchmark/model, materialise them into + the new ``work_dir`` before evalscope starts so the next run resumes from + the larger historical state instead of overwriting it. + """ + try: + predictions_dir = work_dir / 'predictions' / model_name + report_json = work_dir / 'reports' / model_name / f'{benchmark}.json' + restored = perf_backup_module.restore_from_backup( + Path(output_dir), benchmark, model_name, + predictions_dir, report_json, + ) + if restored: + print(f'Restored {benchmark}/{model_name} from backup before run') + return restored + except Exception as e: + print(f'WARNING: perf restore for {benchmark} failed: {e}') + return False + + +def run_and_summarize(task_cfg, write_summary_flag: bool, output_dir: str, model_name: str, summary_name: str): + """Run a benchmark task and optionally refresh the summary table. + + After a successful ``run_task()`` we snapshot the cumulative + ``perf_metrics.summary`` and the per-sample predictions into the durable + backup files maintained by ``perf_backup.py`` so checkpoint restarts can + recover them. + """ + dataset_name = task_cfg.datasets[0] + work_dir = Path(task_cfg.work_dir) + restore_before_run(output_dir, dataset_name, model_name, work_dir) + + start_ts = time.monotonic() + try: + run_task(task_cfg) + finally: + elapsed = time.monotonic() - start_ts + perf_backup_module.record_active_time(output_dir, dataset_name, model_name, elapsed) + print(f'Active time for {dataset_name}: {elapsed:.1f}s (total accumulated)') + + backup_after_run(output_dir, dataset_name, model_name, work_dir) + if write_summary_flag: + write_summary(output_dir, model_name, summary_name) + + +def main(): + parser = build_parser() + args = parser.parse_args() + + # Resolve limit + limit = args.limit + if limit is not None: + if str(limit).lower() in ('none', 'all'): + limit = None + else: + limit = int(limit) + + enable_thinking = DEFAULT_ENABLE_THINKING if args.thinking is None else args.thinking + + # Resolve suite or custom datasets + if args.datasets: + custom = [d.strip() for d in args.datasets.split(',') if d.strip()] + multi_run = [d for d in custom if d in MULTI_RUN_CONFIG] + single_run = [d for d in custom if d not in MULTI_RUN_CONFIG] + agent = [d for d in custom if d in ALL_AGENT] + single_run = [d for d in single_run if d not in ALL_AGENT] + else: + suite = SUITES[args.suite] + multi_run = list(suite['multi']) + single_run = list(suite['single']) + agent = list(suite['agent']) + + # Apply --exclude + if args.exclude: + exclude = {d.strip() for d in args.exclude.split(',') if d.strip()} + multi_run = [d for d in multi_run if d not in exclude] + single_run = [d for d in single_run if d not in exclude] + agent = [d for d in agent if d not in exclude] + + judge_model_args = { + 'model_id': args.judge_model, + 'api_url': args.judge_api_url, + 'api_key': args.judge_api_key, + 'eval_type': 'openai_api', + 'generation_config': { + 'temperature': 0.0, + 'max_tokens': args.judge_max_tokens, + }, + } + + truncation_tokens = args.truncation_tokens + _patch_adapters_for_truncation(args.tokenizer_path, truncation_tokens) + + dataset_configs = load_dataset_configs(args.config) + + print('=' * 60) + print(f'Config: {args.config}') + print(f'Model: {args.model}') + print(f'API URL: {args.api_url}') + print(f'Dataset Dir: {args.dataset_dir}') + print(f'Output Dir: {args.output_dir}') + print(f'Suite: {args.suite}') + print(f'Limit: {limit if limit is not None else "ALL"}') + print(f'Thinking: {enable_thinking}') + print(f'Seed: {args.seed}') + print(f'Batch Size: {args.batch_size}') + print(f'Tokenizer Path: {args.tokenizer_path}') + print(f'Truncation Tokens: {truncation_tokens}') + print(f'Multi-run datasets: {multi_run}') + print(f'Single-run datasets: {single_run}') + print(f'Agent datasets: {agent}') + print(f'Write summary: {args.write_summary}') + print('=' * 60) + + def run_one(dataset_name, run_idx=0): + ds_cfg = dataset_configs[dataset_name] + task_cfg = build_task_config( + dataset_name, ds_cfg, args.batch_size, enable_thinking, args.seed, limit, + args.output_dir, args.model, args.api_url, args.dataset_dir, judge_model_args, + run_idx=run_idx, + ) + try: + run_and_summarize(task_cfg, args.write_summary, args.output_dir, args.model, args.summary_name) + except Exception as e: + print(f'ERROR in {dataset_name} (run {run_idx + 1 if run_idx else 1}): {e}') + + for dataset_name in multi_run: + if dataset_name not in dataset_configs: + print(f'WARNING: {dataset_name} not in YAML config, skipping') + continue + num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) + for run_idx in range(num_runs): + print(f"\n{'='*60}") + print(f'Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={args.seed})') + print(f"{'='*60}") + run_one(dataset_name, run_idx=run_idx) + + for dataset_name in single_run: + if dataset_name not in dataset_configs: + print(f'WARNING: {dataset_name} not in YAML config, skipping') + continue + print(f"\n{'='*60}") + print(f'Running: {dataset_name} (seed={args.seed})') + print(f"{'='*60}") + run_one(dataset_name) + + for dataset_name in agent: + if dataset_name not in dataset_configs: + print(f'WARNING: {dataset_name} not in YAML config, skipping') + continue + print(f"\n{'='*60}") + print(f'Running: {dataset_name} (seed={args.seed})') + print(f"{'='*60}") + run_one(dataset_name) + + if args.write_summary: + write_summary(args.output_dir, args.model, args.summary_name) + print('\nAll benchmarks done!') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/bash/test_e2e.py b/bash/test_e2e.py new file mode 100644 index 0000000..896c5f8 --- /dev/null +++ b/bash/test_e2e.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""End-to-end offline test for bash/run.py + collect_results + perf_backup. + +This script does NOT call any model API. It uses an existing benchmark +directory (aime24/seed_42) and walks through every code path that the real +``run.py`` would touch: + +1. backup_perf_stats — confirm a perf summary snapshot is written +2. archive_predictions — confirm per-sample records are deduplicated into + the archive +3. restore_from_backup — confirm a fresh work_dir can be rehydrated from + the backup files +4. collect_results aggregation — confirm correct metrics are produced +5. Whitelist mode — confirm only the requested benchmarks appear in the + summary +6. hle_low alias — confirm hle_low directory maps to canonical "hle" +7. Reset recovery — confirm that even after wiping the predictions file + and the report summary, the archive+backup still recover the metrics + +Run from the project root: + + python bash/test_e2e.py +""" + +import json +import shutil +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +import collect_results as cr # noqa: E402 +import perf_backup as pb # noqa: E402 + + +PROJECT_ROOT = Path('/data1/sora/evalscope') +SOURCE_OUTPUT = PROJECT_ROOT / 'output' + + +def banner(msg): + print('\n' + '=' * 60) + print(f' {msg}') + print('=' * 60) + + +def assert_close(actual, expected, name, tol=1e-3): + if abs(actual - expected) > tol: + raise AssertionError(f'{name}: expected ~{expected}, got {actual}') + print(f' OK {name}={actual:.4f}') + + +def main(): + banner('Setting up isolated test output dir') + test_root = PROJECT_ROOT / 'output_e2e_test' + if test_root.exists(): + shutil.rmtree(test_root) + test_root.mkdir(parents=True) + + # Copy the canonical aime24 data into the isolated dir + src = SOURCE_OUTPUT / 'aime24' / 'seed_42' + dst = test_root / 'aime24' / 'seed_42' + shutil.copytree(src, dst) + + # And copy hle_low as hle_low under the test dir (so we exercise the + # alias code path). + hle_src = SOURCE_OUTPUT / 'hle_low' / 'seed_42' + hle_dst = test_root / 'hle_low' / 'seed_42' + if hle_src.exists(): + shutil.copytree(hle_src, hle_dst) + + benchmark = 'aime24' + model_name = 'DeepSeek-V4-Flash-Int8' + + banner('1) backup_perf_stats — snapshot summary') + report_json = dst / 'reports' / model_name / f'{benchmark}.json' + backup_path = pb.backup_perf_stats(test_root, benchmark, model_name, report_json) + assert backup_path.exists(), 'backup file was not created' + payload = json.loads(backup_path.read_text()) + assert payload['n_samples'] == 30, f'expected 30 samples, got {payload["n_samples"]}' + print(f' OK backup written: {backup_path}, n_samples={payload["n_samples"]}') + + banner('2) archive_predictions — deduplicate into archive') + preds = dst / 'predictions' / model_name + archive_path = pb.archive_predictions(test_root, benchmark, model_name, preds) + assert archive_path.exists(), 'archive file was not created' + n_lines = sum(1 for line in archive_path.read_text().splitlines() if line.strip()) + assert n_lines == 30, f'expected 30 archived samples, got {n_lines}' + print(f' OK archive written: {archive_path}, samples={n_lines}') + + banner('3) restore_from_backup — rehydrate a fresh work_dir') + fresh_dir = test_root / 'fresh_work' / 'seed_42' + fresh_report = fresh_dir / 'reports' / model_name / f'{benchmark}.json' + fresh_preds = fresh_dir / 'predictions' / model_name + # Pre-create empty structure to mimic what eval_task would do + fresh_report.parent.mkdir(parents=True, exist_ok=True) + fresh_report.write_text(json.dumps({ + 'name': f'{model_name}@{benchmark}', + 'perf_metrics': {'summary': {'n_samples': 1}}, + })) + restored = pb.restore_from_backup( + test_root, benchmark, model_name, fresh_preds, fresh_report, + ) + assert restored, 'restore_from_backup returned False' + restored_report = json.loads(fresh_report.read_text()) + assert restored_report['perf_metrics']['summary']['n_samples'] == 30, \ + f'restored n_samples={restored_report["perf_metrics"]["summary"]["n_samples"]}' + print(' OK restored report n_samples=30 (matches backup)') + + banner('4) collect_results — aggregate metrics') + csv, xlsx = cr.eval_benchmark([benchmark], test_root, model_name) + import pandas as pd + df = pd.read_csv(csv) + aime24 = df[df['Benchmark'] == 'aime24'].iloc[0] + # Single seed_42 run, so values match the report directly (no multi-run + # averaging). The full-summary CSV averages across run_1..run_11. + assert_close(aime24['得分'], 0.6333, 'aime24 score (single seed)') + assert_close(aime24['实测时间(h)'], 0.1469, 'aime24 duration (h)') + assert_close(aime24['总样本数'], 30, 'aime24 sample count') + assert_close(aime24['延迟_mean(s)'], 35.24971, 'aime24 latency mean') + assert_close(aime24['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean') + assert_close(aime24['TPOT_mean(s)'], 0.02165, 'aime24 TPOT mean') + assert_close(aime24['输入tokens_mean'], 119.33, 'aime24 input_tokens_mean') + assert_close(aime24['输出tokens_mean'], 1609.8, 'aime24 output_tokens_mean') + + banner('5) Whitelist mode — only the requested benchmarks appear') + csv2, _ = cr.eval_benchmark([benchmark], test_root, model_name) + df2 = pd.read_csv(csv2) + assert df2['Benchmark'].dropna().tolist() == ['aime24'], \ + f'whitelist did not limit benchmarks: {df2["Benchmark"].tolist()}' + print(' OK summary only contains aime24 + total') + + banner('6) hle_low → hle alias') + if hle_src.exists(): + csv3, _ = cr.eval_benchmark(['hle'], test_root, model_name) + df3 = pd.read_csv(csv3) + assert 'hle' in df3['Benchmark'].tolist(), \ + f'hle canonical name missing from: {df3["Benchmark"].tolist()}' + assert 'hle_low' not in df3['Benchmark'].tolist(), \ + f'hle_low should be aliased away: {df3["Benchmark"].tolist()}' + print(' OK hle_low directory maps to canonical "hle"') + + banner('7) Reset recovery — wipe report+predictions, expect archive to restore') + # Wipe the current predictions + shutil.rmtree(preds) + # Reset report.json + data = json.loads(report_json.read_text()) + data['perf_metrics']['summary']['n_samples'] = 1 + data['perf_metrics']['summary']['latency']['mean'] = 0.0 + report_json.write_text(json.dumps(data, indent=2)) + + csv4, _ = cr.eval_benchmark([benchmark], test_root, model_name) + df4 = pd.read_csv(csv4) + aime24_after = df4[df4['Benchmark'] == 'aime24'].iloc[0] + assert_close(aime24_after['得分'], 0.6333, 'aime24 score (post-reset)') + assert_close(aime24_after['总样本数'], 30, 'aime24 sample count (post-reset)') + assert_close(aime24_after['延迟_mean(s)'], 35.24971, 'aime24 latency mean (post-reset)') + assert_close(aime24_after['TTFT_mean(s)'], 0.26301, 'aime24 TTFT mean (post-reset)') + print(' OK archive fully recovered metrics after predictions wipe') + + banner('8) Backup monotonicity — new backup must not overwrite a larger one') + # We have a backup with n_samples=30. Now write a fresh report with + # n_samples=2 and call backup_perf_stats again — the backup should NOT + # be overwritten. + data = json.loads(report_json.read_text()) + data['perf_metrics']['summary']['n_samples'] = 2 + report_json.write_text(json.dumps(data, indent=2)) + pb.backup_perf_stats(test_root, benchmark, model_name, report_json) + payload2 = json.loads(backup_path.read_text()) + assert payload2['n_samples'] == 30, \ + f'backup was clobbered: n_samples now {payload2["n_samples"]}' + print(' OK backup preserved n_samples=30 even after a reset run') + + banner('All checks passed') + print(f'Test artifacts under {test_root} (kept for inspection)') + + +if __name__ == '__main__': + main() diff --git a/config/dpv4-int8_nothinking.yaml b/config/dpv4-int8_nothinking.yaml index e6e89e9..d8998c8 100644 --- a/config/dpv4-int8_nothinking.yaml +++ b/config/dpv4-int8_nothinking.yaml @@ -191,3 +191,15 @@ bfcl_v3: extra_params: is_fc_model: true underscore_to_dot: true +swe_bench_verified: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +swe_bench_pro: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 \ No newline at end of file diff --git a/myread.md b/myread.md new file mode 100644 index 0000000..42d2ffe --- /dev/null +++ b/myread.md @@ -0,0 +1,629 @@ +# EvalScope 评测手册（DeepSeek-V4-Flash-INT8 no-thinking） + +本手册记录基于 EvalScope 对 `DeepSeek-V4-Flash-INT8`（no-thinking 模式）进行能力评测的完整流程，包括代码/数据/镜像下载、环境搭建、benchmark 选择、参数配置、Docker 一键评测和结果收集。 + +--- + +## 1. 项目结构 + +```text +/data1/sora/evalscope/ +├── bash/ # Python 评测入口脚本 +│ ├── run.py # 唯一主入口 +│ ├── collect_results.py # 结果汇总成 Excel/CSV +│ ├── perf_backup.py # 断点续跑的 perf 持久化备份 +│ └── pull_swe_bench_images.py # 预拉 SWE-bench 实例镜像 +├── scripts/ # Shell 辅助脚本 +│ ├── run_docker_full.sh +│ ├── run_docker_lite.sh +│ └── run_docker_group.sh +├── config/ # 评测参数配置文件 +│ └── dpv4-int8_nothinking.yaml +├── evalscope/ # EvalScope 源码（pip install -e .） +├── datasets/ # 评测数据缓存 +├── output/ # 默认输出目录 +├── docker/ # Docker 镜像/数据根目录 +│ ├── images/ # Docker data-root +│ └── containerd/ # containerd root +├── scritps -> bash/ # 兼容旧路径的软链接 +└── tools/docker/ # Docker 镜像构建相关 +``` + +### 目录职责 + +| 目录 | 作用 | +|------|------| +| `bash/` | 可直接执行的 Python 入口脚本。`run.py` 是统一评测入口，`collect_results.py` 用于结果汇总，`perf_backup.py` 维护断点续跑的持久化备份，`pull_swe_bench_images.py` 用于预拉 SWE-bench 镜像。 | +| `scripts/` | Shell 辅助脚本，例如宿主机一键启动 Docker 容器。 | +| `config/` | YAML 配置文件，集中管理各 benchmark 的 `max_tokens`、`temperature` 等参数。 | +| `evalscope/` | EvalScope 源码，通过 `pip install -e .` 安装。 | +| `datasets/` | 评测数据缓存目录。 | +| `output/` | 默认评测结果输出目录。 | +| `docker/images/` | Docker `data-root`，存放所有拉取的镜像层。 | +| `docker/containerd/` | containerd `root`，存放镜像元数据。 | + +> **注意**：`scritps` 是历史遗留的目录名（拼写错误），已通过软链接指向 `bash/`。请统一使用 `bash/run.py`。 + +--- + +## 2. 资源准备 + +评测前需要准备四类资源：**代码**、**数据**、**Docker 环境镜像**、**Docker 执行镜像**。下面分别说明下载/加载方式。 + +### 2.1 Code（评测入口与配置） + +- 仓库地址：`https://git.meta-stone.net/sora/evalstone.git` +- 推荐版本：`798c3d06ab298d7d6b7ed69915d9b01736653bc7` +- 本地路径：`/data1/sora/evalscope` + +```bash +# 若已克隆，更新到指定 commit +cd /data1/sora/evalscope +git fetch origin +git checkout 798c3d06ab298d7d6b7ed69915d9b01736653bc7 + +# 若首次克隆 +git clone https://git.meta-stone.net/sora/evalstone.git /data1/sora/evalscope +cd /data1/sora/evalscope +git checkout 798c3d06ab298d7d6b7ed69915d9b01736653bc7 +``` + +> 该 commit 之后可能有新提交。建议先用 `git log --oneline -5` 查看最新状态；如无特殊说明，默认使用上述版本。 + +### 2.2 Data（评测数据集） + +数据集托管在 ModelScope：`SoraAmami/evalscope-datasets` + +```bash +pip install modelscope +modelscope login --token ms-3d554a39-6e07-496d-8022-0b0ee64a6389 + +python3 -c " +from modelscope.hub.snapshot_download import snapshot_download +snapshot_download( + 'SoraAmami/evalscope-datasets', + repo_type='dataset', + cache_dir='/data1/sora/evalscope', + local_dir='/data1/sora/evalscope' +) +" +``` + +下载完成后目录结构： + +```text +/data1/sora/evalscope/ +├── datasets/ # benchmark 数据 +│ ├── AI-ModelScope_gsm8k-*/ +│ └── ... +└── ... +``` + +> **不要把 `dataset_dir` 指向已经包含 `datasets/` 子目录的路径**（例如 `/data1/sora/evalscope/datasets`），否则 evalscope 会到 `datasets/datasets/` 下重新查找/下载，造成数据重复。应指向父目录 `/data1/sora/evalscope`。 +> +> 也不要直接用 `MsDataset.load()` 加载整个数据集，因为不同 benchmark 的 schema 不同，会报 `CastError: column names don't match`。 + +### 2.3 Docker 环境镜像（evalscope-complete-py312） + +该镜像包含 Python 3.12 + EvalScope 全部依赖，是主要运行环境。 + +**方式一：从本地 tar.gz 加载（离线/已拷贝）** + +```bash +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz +docker images | grep evalscope-complete-py312 +``` + +**方式二：从 ModelScope 下载** + +```bash +pip install modelscope +modelscope login --token ms-3d554a39-6e07-496d-8022-0b0ee64a6389 + +python3 -c " +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-docker', + file_path='evalscope-complete-py312.tar.gz', + local_dir='/data1/sora/evalscope/docker' +) +" + +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz +docker images | grep evalscope-complete-py312 +``` + +**方式三：离线 tar.gz 拷贝** + +如果目标机器无法访问 ModelScope： + +```bash +# 在源机器导出 +# docker save -o evalscope-complete-py312.tar.gz evalscope-complete-py312:latest + +# 在目标机器加载 +docker load -i /path/to/docker/evalscope-complete-py312.tar.gz +docker images | grep evalscope-complete-py312 +``` + +> 此方式只拷贝 evalscope 运行环境，数据集仍需单独拷贝或下载。 + +### 2.4 Docker 执行镜像（代码 sandbox 与 SWE-bench） + +这些镜像用于在容器内执行被测代码，主要包括： + +| 镜像 | 用途 | +|------|------| +| `bigcodebench/bigcodebench-evaluate:latest` | `bigcodebench` / `humaneval` 官方 sandbox | +| `bigcodebench-sandbox:latest` | 基于官方镜像构建的守护态 sandbox | +| `python:3.11-slim` | 通用代码执行环境 | +| `swebench/sweb.eval.x86_64.*` | SWE-bench 每个样本一个实例镜像 | + +#### 2.4.1 配置 Docker（国内镜像加速 + 大容量存储路径） + +SWE-bench 镜像数量多、体积大，建议把 Docker `data-root` 和 containerd `root` 都迁移到 `/data1` 大容量盘： + +```bash +# 1. 停止 Docker +systemctl stop docker.socket +systemctl stop docker +systemctl stop containerd + +# 2. 创建新的数据目录 +mkdir -p /data1/sora/evalscope/docker/images +mkdir -p /data1/sora/evalscope/docker/containerd + +# 3. 配置国内镜像加速（多填几个，自动轮询） +cat > /etc/docker/daemon.json <<'EOF' +{ + "data-root": "/data1/sora/evalscope/docker/images", + "registry-mirrors": [ + "https://docker.m.daocloud.io", + "https://docker.1ms.run", + "https://docker.1panel.live", + "https://hub.rat.dev", + "https://hub-mirror.c.163.com", + "https://mirror.baidubce.com", + "https://dockerproxy.com", + "https://docker.mirrors.aliyun.com", + "https://docker.mirrors.sjtug.sjtu.edu.cn", + "https://docker.nju.edu.cn", + "https://docker.mirrors.ustc.edu.cn", + "https://docker.mirrors.tuna.tsinghua.edu.cn" + ] +} +EOF + +# 4. 配置 containerd 数据目录 +cat > /etc/containerd/config.toml <<'EOF' +root = "/data1/sora/evalscope/docker/containerd" +state = "/run/containerd" +EOF + +# 5. 重启 +systemctl reset-failed docker.service +systemctl start containerd +systemctl start docker +``` + +#### 2.4.2 拉取/构建代码执行镜像 + +```bash +# bigcodebench 官方镜像 +docker pull bigcodebench/bigcodebench-evaluate:latest + +# 构建守护态 sandbox +docker build -t bigcodebench-sandbox:latest -f - . <<'EOF' +FROM bigcodebench/bigcodebench-evaluate:latest +ENTRYPOINT [] +CMD ["tail", "-f", "/dev/null"] +EOF + +# 通用 Python 执行环境 +docker pull python:3.11-slim +``` + +验证： + +```bash +docker images | grep -E 'evalscope-complete-py312|bigcodebench|python:3.11-slim' +``` + +#### 2.4.3 预拉 SWE-bench 实例镜像（可选，强烈建议） + +SWE-bench 每个样本一个独立 Docker 镜像，评测时再拉会极慢，且容易触发限流。建议提前批量预拉： + +```bash +cd /data1/sora/evalscope +export USE_MODELSCOPE_HUB=1 +export HF_ENDPOINT=https://hf-mirror.com +export PYTHONUNBUFFERED=1 + +# 推荐单 worker，避免触发限流 +python bash/pull_swe_bench_images.py \ + --dataset swe_bench_verified \ + --max-workers 1 +``` + +脚本特性： + +- 断点续传：状态文件 `output/.swe_bench_pull_state.json`，中断后再次运行会自动跳过已成功的镜像。 +- 本地已有镜像自动跳过。 +- 单 worker 模式稳定，减少限流概率；若网络非常好，可适当提高 `--max-workers`。 + +镜像会存到 Docker `data-root` 配置的路径： + +```text +/data1/sora/evalscope/docker/images/ +``` + +> 如果拉取过程中频繁失败，可尝试： +> 1. 换一个 registry-mirror； +> 2. 降低 `--max-workers` 到 1； +> 3. 使用 `tmux` 挂后台运行，避免 SSH 断连导致中断。 + +--- + +## 3. 环境准备 + +### 3.1 方式一：本地 Conda 环境 + +```bash +conda create -n evalscope python=3.12 -y +conda activate evalscope +cd /data1/sora/evalscope +cd evalscope && pip install -e . && cd .. + +# 关键依赖 +pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' +pip install git+https://github.com/sierra-research/tau2-bench@v0.2.0 +pip install openpyxl # 用于结果汇总 +``` + +### 3.2 方式二：Docker 一键环境（推荐多机复用） + +除 `swe_bench` 系列外，其他 benchmark 均可直接运行。 + +**运行容器：** + +```bash +mkdir -p /data1/sora/evalscope/output + +docker run -it --rm \ + --network host \ + -v /data1/sora/evalscope:/opt/evalscope \ + -v /data1/models:/opt/models \ + -v /var/run/docker.sock:/var/run/docker.sock \ + evalscope-complete-py312:latest \ + bash + +# 容器内 +cd /opt/evalscope +python bash/run.py --help +``` + +> **数据集路径注意**：evalscope 会在 `--dataset-dir` 后自动拼接 `datasets/` 子目录查找数据。若数据在 `/data1/sora/evalscope/datasets/`，则 `--dataset-dir` 应填 `/data1/sora/evalscope`，不要填 `/data1/sora/evalscope/datasets`。 + +--- + +## 4. 统一入口：`bash/run.py` + +所有评测都通过这一个脚本启动，全部参数均可通过命令行控制。 + +### 4.1 常用示例 + +```bash +cd /data1/sora/evalscope + +# 1) Full 全量评测（约 75h，CSV 实测完整时间，已含 multi-run） +python bash/run.py \ + --model DeepSeek-V4-Flash-Int8 \ + --api-url http://localhost:30000/v1 \ + --dataset-dir /data1/sora/evalscope \ + --output-dir /data1/sora/evalscope/output \ + --suite full \ + --limit none + +# 2) Lite 快速冒烟（约 5h） +python bash/run.py --suite lite --limit none + +# 3) Lite 自己选 benchmark +python bash/run.py \ + --datasets humaneval,aime24,gsm8k,arc,longbench_v2,general_fc \ + --limit 20 + +# 4) 只跑某几个 benchmark +python bash/run.py --datasets aime24,gsm8k --limit none + +# 5) 多机分组：group1 / group2 / group3 +python bash/run.py --suite group1 --limit none +python bash/run.py --suite group2 --limit none +python bash/run.py --suite group3 --limit none + +# 6) 从某个 suite 中排除特定 benchmark +python bash/run.py --suite full --exclude hle,trivia_qa --limit none + +# 7) 开启 thinking 模式 +python bash/run.py --suite lite --thinking + +# 8) 换模型或 API 地址 +python bash/run.py --model MyModel --api-url http://10.0.0.5:30000/v1 +``` + +### 4.2 参数说明 + +| 参数 | 默认值 | 说明 | +|------|--------|------| +| `--model` | `DeepSeek-V4-Flash-Int8` | 模型名，会传给 OpenAI API 的 `model` 字段 | +| `--api-url` | `http://localhost:30000/v1` | 模型服务地址 | +| `--dataset-dir` | `/data1/sora/evalscope` | 数据集父目录，evalscope 会自动找其下的 `datasets/` | +| `--output-dir` | `/data1/sora/evalscope/output` | 评测输出根目录 | +| `--config` | `config/dpv4-int8_nothinking.yaml` | 各 benchmark 的 `max_tokens` / `temperature` 等配置 | +| `--tokenizer-path` | `/data1/models/DeepSeek-V4-Flash-INT8` | 本地 tokenizer 路径，用于长文本 middle-truncation | +| `--suite` | `full` | 预置套件：`full` / `lite` / `mid` / `group1` / `group2` / `group3` | +| `--datasets` / `--benchmarks` | `None` | 逗号分隔自定义 benchmark 列表，会覆盖 `--suite` | +| `--exclude` | `None` | 从当前 suite 中排除的 benchmark | +| `--limit` | `None` | 每数据集最多跑几条；`none` / `all` 表示全量 | +| `--seed` | `42` | 随机种子 | +| `--batch-size` | `4` | 评测并发 | +| `--thinking` | `False` | 开启 sglang thinking 模式 | +| `--no-thinking` | `False` | 关闭 thinking 模式（默认） | +| `--judge-model` | `DeepSeek/DeepSeek-V4-Pro` | 裁判模型名（LLM-as-judge） | +| `--judge-api-url` | Vectron 地址 | 裁判模型 API 地址 | +| `--judge-api-key` | 内置 key | 裁判模型 API key | +| `--judge-max-tokens` | `10240` | 裁判模型最大输出长度 | +| `--truncation-tokens` | `131072` | 长文本 middle-truncation token 上限 | +| `--no-summary` | `False` | 禁止每次 benchmark 后自动写汇总表 | + +> 汇总表文件名由模型名自动派生，例如 `--model DeepSeek-V4-Flash-Int8` → `.csv/.xlsx`。 + +### 4.3 套件详情 + +| 套件 | 覆盖 benchmark | 预计时间 | 说明 | +|------|---------------|----------|------| +| `full` | 全部 | 约 **75h** | CSV 实测完整时间，已含 multi-run | +| `lite` | 各领域代表作 | 约 **5h** | 快速冒烟，覆盖代码/数学/知识/长文本/Agent | +| `mid` | 较全子集 | 约 **25h** | 平衡速度与覆盖 | +| `group1` | 代码 + 数学/推理 | 约 **22h** | 多机组 1 | +| `group2` | 重知识 | 约 **26h** | 多机组 2 | +| `group3` | 长文本 + Agent + 知识 | 约 **27h** | 多机组 3 | + +> **分组时间说明** +> +> CSV 中记录的是**完整评测时间**（已包含 multi-run），总约 75h。因此分组直接按 CSV 实测时间平衡：Group1 ~22h、Group2 ~26h、Group3 ~27h，合计 ~75h。 + +### 4.4 Multi-run 配置 + +以下 benchmark 会重复跑多次，再取平均： + +| Benchmark | 次数 | 目的 | +|-----------|------|------| +| `aime24` / `aime25` / `aime26` / `hmmt26` | 12 | 样本少（各 30 题），多次采样稳定结果 | +| `live_code_bench` | 5 | 约 100 样本，多次采样 | +| `imo_answerbench` | 4 | 约 120 样本 | +| `humaneval` | 3 | 约 164 样本 | +| `gpqa_diamond` | 2 | 约 198 样本 | + +multi-run 次数在 `bash/run.py` 的 `MULTI_RUN_CONFIG` 中固定，暂不提供命令行覆盖。 + +--- + +## 5. 配置文件 + +路径：`config/dpv4-int8_nothinking.yaml` + +每个 benchmark 可独立配置： + +```yaml +aime24: + generation_config: + temperature: 1.0 # 多次采样用 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 + +longbench_v2: + generation_config: + temperature: 0.0 # 长文本贪心 + top_p: 1.0 + stream: true + max_tokens: 8192 + dataset_args: + subset_list: + - short + - medium + - long +``` + +如需换模型，通常只需改 `bash/run.py` 的命令行参数，不需要改 YAML。 + +--- + +## 6. 长文本截断 + +`longbench_v2` 和 `openai_mrcr` 默认启用 **middle-truncation**，上限由 `--truncation-tokens` 控制（默认 128k）。 + +- 对 `longbench_v2`：保留 context 头部和尾部，截断中间。 +- 对 `openai_mrcr`：保留对话头尾及 needle 附近窗口，并限制单条消息长度。 + +如需关闭截断： + +```bash +python bash/run.py --suite full --truncation-tokens 0 +``` + +--- + +## 7. 结果汇总：`bash/collect_results.py` + +跑完 benchmark 后，自动或手动汇总分数到 Excel/CSV。 + +### 7.1 自动汇总（run.py 每次跑完都会更新） + +`run.py` 会在每个 benchmark 结束后自动调用 `collect_results.py`，在按模型名分目录的位置生成： + +```text +/ +└── / # 按模型名分目录，目录和文件名均由 --model 自动派生 + ├── .xlsx + └── .csv +``` + +例：`--model DeepSeek-V4-Flash-Int8` → `/data1/sora/evalscope/output/DeepSeek-V4-Flash-Int8/DeepSeek-V4-Flash-Int8.csv`。 + +### 7.2 手动汇总 + +```bash +# 汇总所有已跑完的 benchmark（默认行为） +python bash/collect_results.py \ + --output-dir /data1/sora/evalscope/output \ + --model DeepSeek-V4-Flash-Int8 + +# 只汇总指定 benchmark（推荐，跟 run.py 自动汇总行为一致） +python bash/collect_results.py \ + --output-dir /data1/sora/evalscope/output \ + --model DeepSeek-V4-Flash-Int8 \ + --include aime24,gsm8k,arc,longbench_v2 + +# 可选：--out-name 自定义文件名（默认就是 safe model name） +``` + +### 7.3 汇总规则 + +- **只统计本次实际跑的 benchmark**：`run.py` 会维护一个白名单，只把当前这次运行涉及的 benchmark 传给汇总脚本。因此，如果本次只跑了 `aime24,gsm8k,arc`，汇总表里也只会出现这 3 个，不会把 output 目录里以前跑的其它 benchmark 一起拉进来。 +- 对同一个 benchmark 的多个 seed/run，**分数取平均**。 +- 性能指标（latency、TTFT、TPOT、TPS 等）**从所有 run 的原始 `predictions/*.jsonl` 重新聚合**，避免断点续跑时从 0 重置。 +- 输出格式与 `/data1/sora/evalscope/P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv` 保持一致。 + +### 7.4 断点续跑统计修复 + +**问题**：`evalscope` 在运行时会把所有样本的累计统计写入 `reports/*.json` 里的 `perf_metrics.summary`（旧版本会写到独立的 `perf_stats.json`），并在 `predictions/*.jsonl` 里记录每条样本的原始 perf 数据。如果一次 benchmark 需要跑很久，中间断点续跑时这个累计 summary 会被**重新初始化**（只反映本次重启之后的样本），导致汇总表里 latency / TTFT / TPOT / 总 tokens / 总耗时全部**从 0 开始重新计算**，出现严重低估。 + +**修复思路**：保留 `report.json` 里的累计 summary，但**额外维护两份独立于 evalscope 的持久化文件**： + +| 文件 | 内容 | 用途 | +|------|------|------| +| `perf_stats_backup/__.json` | 最近一次完整跑完的 `perf_metrics.summary` | 累计 summary 的稳定副本，避免被重启覆盖 | +| `predictions_archive/__.jsonl` | 历次 run 所有样本的原始记录，按 `index` 去重 | 重建 latency / TTFT / TPOT 的真实分布（含 P90/P99） | + +每次 `run_task()` 完成后，`run.py` 会自动调用 `bash/perf_backup.py`： + +1. 把当前 `reports//.json` 里的 `perf_metrics.summary` 复制到 `perf_stats_backup/__.json`。如果 backup 的 `n_samples` 已经大于新 summary（即检测到 reset），则保留旧 backup，避免覆盖。 +2. 把当前 `predictions//*.jsonl` 的每条样本按 `index` 去重追加到 `predictions_archive/__.jsonl`，支持多次断点的样本累积。 + +每次下一个 run 开始前，`run.py` 会调用 `restore_from_backup()`：如果 backup 存在，就把 archive 还原回 `predictions//` 并把 summary 还原回 `report.json` 的 `perf_metrics.summary`，让 evalscope 的 `use_cache` resume 机制能从更大的历史状态继续。 + +`collect_results.py` 读取汇总时的优先级： + +1. **优先**读 `predictions_archive/__.jsonl`，按 `index` 去重得到所有历史样本，从原始 perf_metrics 重新计算 latency / TTFT / TPOT 的 mean、P90、P99、QPS、TPS、总 tokens。 +2. 没有 archive 时，读 `predictions//*.jsonl`（当前 predictions）。 +3. 都没有时，回退到 `perf_stats_backup/__.json` 的 summary。 +4. summary 里 `n_samples` 比 backup 大就用 summary，否则用 backup 里的累计值。 + +### 7.5 验证断点续跑确实能恢复 + +```python +# 1. 模拟 report.json 被重置（n_samples 改成 2） +# 2. 不动 predictions 和 archive +# 3. 跑 collect_results +# 期望：汇总表仍显示 30 个样本的真实指标 +``` + +实际验证结果：把 `aime24/report.json` 的 `n_samples` 改成 2 后再汇总，得分仍是 **0.6167**、耗时 **1.22h**、**TTFT/TPOT 正常值**，证明 archive 修复确实生效。 + +--- + +## 8. 输出目录结构 + +```text +/ +├── / +│ ├── seed_/ +│ │ ├── reports//.json +│ │ ├── predictions//.jsonl +│ │ └── logs/eval_log.log +│ └── seed__run_/ # multi-run 额外目录 +├── perf_stats_backup/ # 累计 summary 持久化（断点恢复） +│ └── __.json +├── predictions_archive/ # 原始每条样本归档（断点恢复） +│ └── __.jsonl +└── / # 汇总表（按模型名） + ├── .xlsx + └── .csv +``` + +### 查看分数 + +```bash +# 单个 benchmark +cat output/aime24/seed_42/reports/DeepSeek-V4-Flash-Int8/aime24.json + +# 汇总所有分数 +python3 -c " +import json, glob +for f in sorted(glob.glob('output/*/seed_*/reports/*/*.json')): + data = json.load(open(f)) + score = data.get('score', data.get('mean_acc', 'N/A')) + print(f'{f}: {score}') +" +``` + +--- + +## 9. 最终 Benchmark 集合 + +| 能力域 | Benchmark | +|--------|-----------| +| 代码生成 | `humaneval`, `live_code_bench`, `bigcodebench` | +| 推理/数学 | `aime24`, `aime25`, `aime26`, `hmmt26`, `imo_answerbench`, `gsm8k`, `competition_math`, `bbh`, `drop` | +| 知识 | `gpqa_diamond`, `hle`, `mmlu_pro`, `simple_qa`, `super_gpqa`, `mmlu`, `cmmlu`, `arc`, `hellaswag`, `trivia_qa`, `winogrande` | +| 长上下文 | `longbench_v2`, `openai_mrcr` | +| 智能体/工具 | `tau2_bench`, `general_fc`, `bfcl_v3` | + +未纳入标准流程的： + +- `swe_bench_pro` / `swe_bench_verified` / `swe_bench_multilingual_agentic`：每个样本需独立 Docker 镜像，建议按 2.4.3 预拉后单独运行。 +- `browsecomp` / `mcp_atlas`：需要外部 MCP 搜索工具。 +- `terminal_bench_v2`：镜像兼容性问题。 + +--- + +## 10. 常见问题 + +### Q1: Docker 里找不到 datasets？ + +确保 `--dataset-dir` 指向包含 `datasets/` 的**父目录**，而不是 `datasets/` 本身。 + +### Q2: `bigcodebench` 或 `humaneval` 报 sandbox 错误？ + +检查是否已准备镜像： + +```bash +docker images | grep -E 'bigcodebench-sandbox|python:3.11-slim' +``` + +### Q3: 只想跑某个 benchmark 怎么办？ + +```bash +python bash/run.py --datasets aime24 --limit none +``` + +### Q4: 想换 judge model？ + +```bash +python bash/run.py \ + --judge-model deepseek-v4-pro \ + --judge-api-url https://api.deepseek.com/v1 \ + --judge-api-key sk-xxx +``` + +### Q5: 断点续跑后 perf 指标被重置？ + +`run.py` 会在每次 benchmark 后把 `perf_metrics.summary` 和 `predictions` 备份到 `output/perf_stats_backup/` 和 `output/predictions_archive/`；汇总时优先从 archive 重新计算，因此断点重启不会导致指标被低估。同时汇总表只包含**本次运行实际跑的 benchmark**。 + +### Q6: SWE-bench 镜像拉取慢/被限流？ + +1. 按 2.4.1 配置多 mirror； +2. 使用 `--max-workers 1` 单 worker 拉取； +3. 用 `tmux` 或 `nohup` 挂后台，避免 SSH 断连； +4. 若某个 mirror 完全不可用，从 `daemon.json` 中移除，避免无效重试。