#!/usr/bin/env python3
"""
Collect benchmark results from an EvalScope output directory and write a summary
Excel/CSV similar to:

  P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv

Rules:
- Scores are averaged across seeds / multi-runs.
- Perf metrics (latency, TTFT, TPOT, TPS, tokens) are recomputed from raw
  predictions across all seeds / multi-runs, so resuming from a checkpoint
  no longer resets cumulative statistics.
"""

import argparse
import json
import statistics
from pathlib import Path

import numpy as np
import pandas as pd


# Map benchmark -> capability domain (same as the reference CSV)
BENCHMARK_DOMAIN = {
    'bigcodebench': '代码与工程',
    'humaneval': '代码与工程',
    'live_code_bench': '代码与工程',
    'aime24': '推理与数学',
    'aime25': '推理与数学',
    'aime26': '推理与数学',
    'hmmt26': '推理与数学',
    'imo_answerbench': '推理与数学',
    'gsm8k': '推理与数学',
    'competition_math': '推理与数学',
    'bbh': '推理与数学',
    'drop': '推理与数学',
    'gpqa_diamond': '知识与语言理解',
    'hle': '知识与语言理解',
    'hle': '知识与语言理解',
    'hle_low': '知识与语言理解',
    'mmlu_pro': '知识与语言理解',
    'simple_qa': '知识与语言理解',
    'super_gpqa': '知识与语言理解',
    'mmlu': '知识与语言理解',
    'cmmlu': '知识与语言理解',
    'arc': '知识与语言理解',
    'hellaswag': '知识与语言理解',
    'trivia_qa': '知识与语言理解',
    'winogrande': '知识与语言理解',
    'longbench_v2': '长上下文',
    'openai_mrcr': '长上下文',
    'tau2_bench': '智能体与工具',
    'general_fc': '智能体与工具',
    'bfcl_v3': '智能体与工具',
}

# Column order matching the reference CSV
OUTPUT_COLUMNS = [
    '分类',
    'Benchmark',
    '得分',
    '实测时间(h)',
    '总样本数',
    '延迟_mean(s)',
    '输出TPS',
    '请求QPS',
    '输入tokens_mean',
    '输出tokens_mean',
    '累计总tokens',
    'TTFT_mean(s)',
    'TTFT P90',
    'TTFT P99',
    'TPOT_mean(s)',
    'TPOT P90',
    'TPOT P99',
]

# Some evalscope output directory names do not match the canonical benchmark
# name (e.g. the hle benchmark writes to the `hle_low` directory because of
# subset naming). Map them to the canonical name here so the summary uses a
# consistent label.
BENCHMARK_NAME_ALIAS = {
    'hle_low': 'hle',
}


def percentile(values, q):
    """Return the q-th percentile using numpy's linear interpolation."""
    if not values:
        return np.nan
    return float(np.percentile(values, q))


def read_predictions(pred_file: Path):
    """Yield perf_metrics dicts from a predictions JSONL file."""
    for obj in read_predictions_with_index(pred_file):
        if obj['perf_metrics'] is not None:
            yield obj['perf_metrics']


def read_predictions_with_index(pred_file: Path):
    """Yield ``{'index', 'perf_metrics'}`` dicts from a predictions JSONL file."""
    if not pred_file.exists():
        return
    with open(pred_file, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError:
                continue
            model_output = obj.get('model_output', {})
            pm = model_output.get('perf_metrics')
            if not pm and 'choices' in model_output:
                choices = model_output['choices']
                if choices and 'message' in choices[0]:
                    pm = choices[0]['message'].get('perf_metrics')
            yield {'index': obj.get('index'), 'perf_metrics': pm}


def load_backup_summary(output_dir: Path, benchmark: str, model_name: str):
    """Load the durable ``perf_stats_backup/<benchmark>__<model>.json``.

    Returns the ``summary`` dict (with ``latency`` / ``ttft`` / ``tpot`` /
    ``throughput`` / ``usage`` sub-dicts) or ``None`` if no backup exists.
    """
    try:
        from perf_backup import get_backup_paths
    except ImportError:
        return None
    perf_path, _ = get_backup_paths(Path(output_dir), benchmark, model_name)
    if not perf_path.exists():
        return None
    try:
        payload = json.loads(perf_path.read_text(encoding='utf-8'))
        return payload.get('summary')
    except Exception:
        return None


def find_report(output_dir: Path, benchmark: str, model_name: str):
    """Find the first report JSON for a benchmark/model under output_dir."""
    bench_dir = output_dir / benchmark
    if not bench_dir.exists():
        return None
    for seed_dir in sorted(bench_dir.iterdir()):
        if not seed_dir.is_dir():
            continue
        for candidate in find_report_candidates(seed_dir, benchmark, model_name):
            if candidate.exists():
                return candidate
    return None


def find_report_candidates(seed_dir: Path, benchmark: str, model_name: str):
    """Return candidate report paths, accepting variant naming patterns."""
    # The actual report filename may not match the directory name exactly
    # (e.g. hle_low directory holds hle.json). Try all .json files under
    # the model directory.
    candidates = [
        seed_dir / 'reports' / model_name / f'{benchmark}.json',
        seed_dir / 'reports' / f'{benchmark}.json',
    ]
    for p in (seed_dir / 'reports' / model_name).glob('*.json'):
        candidates.append(p)
    for p in (seed_dir / 'reports').glob('*.json'):
        candidates.append(p)
    return candidates


def find_all_reports(output_dir: Path, benchmark: str, model_name: str):
    """Find all report JSONs for a benchmark/model (across seeds/runs)."""
    bench_dir = output_dir / benchmark
    reports = []
    if not bench_dir.exists():
        return reports
    for seed_dir in sorted(bench_dir.iterdir()):
        if not seed_dir.is_dir():
            continue
        for candidate in find_report_candidates(seed_dir, benchmark, model_name):
            if candidate.exists() and candidate not in reports:
                reports.append(candidate)
                break
    return reports


def find_archive_predictions(output_dir: Path, benchmark: str, model_name: str):
    """Return the durable per-sample archive maintained by ``perf_backup.py``.

    The archive contains every sample ever produced across all runs and
    breakpoints for this benchmark/model (deduplicated by ``index``). When
    present, the summary aggregator should prefer it over the raw
    ``predictions/*.jsonl`` files because the latter can be overwritten on
    restart.
    """
    try:
        from perf_backup import get_backup_paths
    except ImportError:
        return []
    _, archive_path = get_backup_paths(output_dir, benchmark, model_name)
    if archive_path.exists():
        return [archive_path]
    return []


def find_all_predictions(output_dir: Path, benchmark: str, model_name: str):
    """Find all predictions JSONL files for a benchmark/model.

    The durable ``predictions_archive/<benchmark>__<model>.jsonl`` (if
    present) is returned **first** so its deduplicated per-sample records
    dominate any smaller predictions that a fresh run may have written.
    """
    files = list(find_archive_predictions(output_dir, benchmark, model_name))
    bench_dir = output_dir / benchmark
    if not bench_dir.exists():
        return files
    for seed_dir in sorted(bench_dir.iterdir()):
        if not seed_dir.is_dir():
            continue
        pred_dir = seed_dir / 'predictions' / model_name
        if pred_dir.exists():
            files.extend(sorted(pred_dir.rglob('*.jsonl')))
    return files


def parse_log_duration(log_file: Path):
    """Parse first and last timestamp from eval_log.log and return duration in hours."""
    if not log_file.exists():
        return np.nan
    from datetime import datetime
    first_dt = None
    last_dt = None
    fmt = '%Y-%m-%d %H:%M:%S'
    with open(log_file, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if len(line) < 19:
                continue
            try:
                dt = datetime.strptime(line[:19], fmt)
            except ValueError:
                continue
            if first_dt is None:
                first_dt = dt
            last_dt = dt
    if first_dt is None or last_dt is None or last_dt <= first_dt:
        return np.nan
    return (last_dt - first_dt).total_seconds() / 3600.0


def extract_score(report_data: dict) -> float:
    """Extract the top-level score from a report JSON."""
    score = report_data.get('score')
    if score is not None:
        return float(score)
    metrics = report_data.get('metrics', [])
    for m in metrics:
        if m.get('name') == 'mean_acc':
            return float(m.get('score', m.get('macro_score', 0.0)))
    return 0.0


def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
    """Collect aggregated results for one benchmark."""
    reports = find_all_reports(output_dir, benchmark, model_name)
    if not reports:
        return None

    scores = []
    summary0 = None
    n_samples_unique = 0
    for report in reports:
        try:
            data = json.loads(report.read_text(encoding='utf-8'))
            scores.append(extract_score(data))
            if summary0 is None:
                summary0 = data.get('perf_metrics', {}).get('summary', {})
                n_samples_unique = summary0.get('n_samples', 0)
        except Exception:
            continue

    avg_score = float(np.mean(scores)) if scores else 0.0

    # Aggregate raw prediction perf metrics across all seeds/runs.
    # This fixes the breakpoint-resume issue where cumulative stats are reset.
    # Predictions are deduplicated by sample `index` so the archive (which
    # spans every run) and the latest ``predictions/*.jsonl`` don't double
    # count the same sample.
    pred_files = find_all_predictions(output_dir, benchmark, model_name)
    seen_indexes = set()
    latencies = []
    ttfts = []
    tpots = []
    input_tokens = []
    output_tokens = []
    sample_indexes = []
    for pf in pred_files:
        for obj in read_predictions_with_index(pf):
            idx = obj['index']
            if idx is None or idx in seen_indexes:
                # Still keep perf data even when we lack an index, so older
                # benchmark files without `index` don't get dropped.
                pm = obj['perf_metrics']
            else:
                seen_indexes.add(idx)
                sample_indexes.append(idx)
                pm = obj['perf_metrics']
            if pm is None:
                continue
            if 'latency' in pm:
                latencies.append(float(pm['latency']))
            if 'ttft' in pm:
                ttfts.append(float(pm['ttft']))
            if 'tpot' in pm:
                tpots.append(float(pm['tpot']))

            itok = pm.get('input_tokens')
            otok = pm.get('output_tokens')
            if itok is None and 'usage' in pm:
                itok = pm['usage'].get('input_tokens')
                otok = pm['usage'].get('output_tokens')
            if itok is not None:
                input_tokens.append(int(itok))
            if otok is not None:
                output_tokens.append(int(otok))

    # If we don't have raw predictions but have a perf_stats backup, that
    # represents a known-good summary captured right after a clean run —
    # preferable to summary0 (which may be the just-reset run).
    backup_summary = None
    if not latencies:
        backup_summary = load_backup_summary(output_dir, benchmark, model_name)

    if latencies:
        latency_mean = float(np.mean(latencies))
        total_compute_time = float(np.sum(latencies))
        total_output_tokens = sum(output_tokens)
        avg_output_tps = total_output_tokens / total_compute_time if total_compute_time > 0 else np.nan
        avg_req_ps = len(latencies) / total_compute_time if total_compute_time > 0 else np.nan
        input_tok_mean = float(np.mean(input_tokens)) if input_tokens else np.nan
        output_tok_mean = float(np.mean(output_tokens)) if output_tokens else np.nan
        total_tokens = sum(input_tokens) + sum(output_tokens)
        ttft_mean = float(np.mean(ttfts)) if ttfts else np.nan
        ttft_p90 = percentile(ttfts, 90)
        ttft_p99 = percentile(ttfts, 99)
        tpot_mean = float(np.mean(tpots)) if tpots else np.nan
        tpot_p90 = percentile(tpots, 90)
        tpot_p99 = percentile(tpots, 99)
        # The actual sample count we just rebuilt from raw predictions is more
        # reliable than the (possibly reset) report summary's n_samples.
        if sample_indexes:
            n_samples_unique = len(sample_indexes)
    elif summary0:
        # Fallback to report summary if raw predictions are unavailable
        latency_mean = summary0.get('latency', {}).get('mean', np.nan)
        avg_output_tps = summary0.get('throughput', {}).get('avg_output_tps', np.nan)
        avg_req_ps = summary0.get('throughput', {}).get('avg_req_ps', np.nan)
        input_tok_mean = summary0.get('tokens', {}).get('input_tokens', {}).get('mean', np.nan)
        output_tok_mean = summary0.get('tokens', {}).get('output_tokens', {}).get('mean', np.nan)
        total_tokens = summary0.get('tokens', {}).get('total_tokens_count', np.nan)
        ttft_mean = summary0.get('ttft', {}).get('mean', np.nan)
        ttft_p90 = summary0.get('ttft', {}).get('90%', np.nan)
        ttft_p99 = summary0.get('ttft', {}).get('99%', np.nan)
        tpot_mean = summary0.get('tpot', {}).get('mean', np.nan)
        tpot_p90 = summary0.get('tpot', {}).get('90%', np.nan)
        tpot_p99 = summary0.get('tpot', {}).get('99%', np.nan)
        # If the just-read report looks like a freshly-reset run (smaller
        # n_samples than the durable backup), prefer the backup's summary so
        # the cumulative numbers are not lost.
        try:
            from perf_backup import get_backup_paths
            backup_path, _ = get_backup_paths(output_dir, benchmark, model_name)
            if backup_path.exists():
                payload = json.loads(backup_path.read_text(encoding='utf-8'))
                if (payload.get('n_samples') or 0) > (summary0.get('n_samples') or 0):
                    backup_summary = payload.get('summary') or {}
                    latency_mean = backup_summary.get('latency', {}).get('mean', latency_mean)
                    avg_output_tps = backup_summary.get('throughput', {}).get('avg_output_tps', avg_output_tps)
                    avg_req_ps = backup_summary.get('throughput', {}).get('avg_req_ps', avg_req_ps)
                    input_tok_mean = backup_summary.get('usage', {}).get('input_tokens', {}).get('mean', input_tok_mean)
                    output_tok_mean = backup_summary.get('usage', {}).get('output_tokens', {}).get('mean', output_tok_mean)
                    total_tokens = backup_summary.get('usage', {}).get('total_tokens_count', total_tokens)
                    ttft_mean = backup_summary.get('ttft', {}).get('mean', ttft_mean)
                    ttft_p90 = backup_summary.get('ttft', {}).get('90%', ttft_p90)
                    ttft_p99 = backup_summary.get('ttft', {}).get('99%', ttft_p99)
                    tpot_mean = backup_summary.get('tpot', {}).get('mean', tpot_mean)
                    tpot_p90 = backup_summary.get('tpot', {}).get('90%', tpot_p90)
                    tpot_p99 = backup_summary.get('tpot', {}).get('99%', tpot_p99)
                    n_samples_unique = payload.get('n_samples') or n_samples_unique
        except Exception:
            pass
    elif backup_summary:
        latency_mean = backup_summary.get('latency', {}).get('mean', np.nan)
        avg_output_tps = backup_summary.get('throughput', {}).get('avg_output_tps', np.nan)
        avg_req_ps = backup_summary.get('throughput', {}).get('avg_req_ps', np.nan)
        input_tok_mean = backup_summary.get('usage', {}).get('input_tokens', {}).get('mean', np.nan)
        output_tok_mean = backup_summary.get('usage', {}).get('output_tokens', {}).get('mean', np.nan)
        total_tokens = backup_summary.get('usage', {}).get('total_tokens_count', np.nan)
        ttft_mean = backup_summary.get('ttft', {}).get('mean', np.nan)
        ttft_p90 = backup_summary.get('ttft', {}).get('90%', np.nan)
        ttft_p99 = backup_summary.get('ttft', {}).get('99%', np.nan)
        tpot_mean = backup_summary.get('tpot', {}).get('mean', np.nan)
        tpot_p90 = backup_summary.get('tpot', {}).get('90%', np.nan)
        tpot_p99 = backup_summary.get('tpot', {}).get('99%', np.nan)
    else:
        return None

    # Duration: prefer the durable active timer maintained by perf_backup.py,
    # which only counts time when run_task() is actually executing. This avoids
    # counting idle gaps caused by manual interruption/resume.
    # If no active timer exists, fall back to log duration or latency estimate.
    active_time_hours = np.nan
    try:
        from perf_backup import load_active_time
        active_seconds = load_active_time(output_dir, benchmark, model_name)
        if active_seconds > 0:
            active_time_hours = active_seconds / 3600.0
    except Exception:
        pass

    if not np.isnan(active_time_hours):
        duration_hours = active_time_hours
    else:
        # Legacy fallback: sum of wall-clock durations from eval_log.log.
        # For sandbox benchmarks the main log may not cover sandbox execution,
        # so we also estimate wall time as compute_time/batch_size and take max.
        duration_hours = 0.0
        batch_size = 4
        for report in reports:
            log_file = report.parent.parent.parent / 'logs' / 'eval_log.log'
            d = parse_log_duration(log_file)
            if not np.isnan(d):
                duration_hours += d
            cfg_file = report.parent.parent.parent / 'configs' / 'task_config.yaml'
            if cfg_file.exists():
                try:
                    import yaml
                    cfg = yaml.safe_load(cfg_file.read_text(encoding='utf-8'))
                    batch_size = int(cfg.get('eval_batch_size', batch_size))
                except Exception:
                    pass

        compute_wall_estimate = (total_compute_time / batch_size / 3600.0) if latencies and batch_size > 0 else np.nan
        if duration_hours <= 0:
            duration_hours = compute_wall_estimate
        else:
            duration_hours = max(duration_hours, compute_wall_estimate)

    return {
        '分类': BENCHMARK_DOMAIN.get(benchmark, '其他'),
        'Benchmark': BENCHMARK_NAME_ALIAS.get(benchmark, benchmark),
        '得分': round(avg_score, 4),
        '实测时间(h)': round(duration_hours, 4) if not np.isnan(duration_hours) else np.nan,
        '总样本数': n_samples_unique,
        '延迟_mean(s)': round(latency_mean, 5) if not np.isnan(latency_mean) else np.nan,
        '输出TPS': round(avg_output_tps, 2) if not np.isnan(avg_output_tps) else np.nan,
        '请求QPS': round(avg_req_ps, 4) if not np.isnan(avg_req_ps) else np.nan,
        '输入tokens_mean': round(input_tok_mean, 2) if not np.isnan(input_tok_mean) else np.nan,
        '输出tokens_mean': round(output_tok_mean, 2) if not np.isnan(output_tok_mean) else np.nan,
        '累计总tokens': total_tokens if not np.isnan(total_tokens) else np.nan,
        'TTFT_mean(s)': round(ttft_mean, 5) if not np.isnan(ttft_mean) else np.nan,
        'TTFT P90': round(ttft_p90, 5) if not np.isnan(ttft_p90) else np.nan,
        'TTFT P99': round(ttft_p99, 5) if not np.isnan(ttft_p99) else np.nan,
        'TPOT_mean(s)': round(tpot_mean, 5) if not np.isnan(tpot_mean) else np.nan,
        'TPOT P90': round(tpot_p90, 5) if not np.isnan(tpot_p90) else np.nan,
        'TPOT P99': round(tpot_p99, 5) if not np.isnan(tpot_p99) else np.nan,
    }


def eval_benchmark(benchmark_names: list, output_dir: Path, model_name: str,
                   out_name: str = None, excel_output_dir: Path = None):
    """Collect results for a specific list of benchmarks.

    This is the public entry point intended for use by ``run.py`` and other
    scripts that already know which benchmarks they ran.

    Args:
        benchmark_names: List of canonical benchmark names to aggregate (e.g.
            ``['aime24', 'gsm8k', 'arc']``). Only benchmarks with reports on
            disk will appear in the summary.
        output_dir: EvalScope output root directory.
        model_name: Model name to look up reports/predictions for.
        out_name: Output file name (without extension). Defaults to the safe
            model name.
        excel_output_dir: Optional separate directory for the Excel summary.
            If provided, the Excel file is written here while the CSV stays
            under ``output_dir/results/<model>/``.

    Returns:
        (csv_path, xlsx_path) tuple.
    """
    return collect_all(output_dir, model_name, out_name,
                       include_benchmarks=benchmark_names,
                       excel_output_dir=excel_output_dir)


def collect_all(output_dir: Path, model_name: str, out_name: str = None,
                include_benchmarks: list = None,
                excel_output_dir: Path = None):
    """Collect benchmarks under ``output_dir`` and write summary Excel/CSV."""
    if not output_dir.exists():
        raise FileNotFoundError(f'Output directory not found: {output_dir}')

    safe_model = model_name.replace('/', '_').replace('\\', '_').replace(' ', '_')

    # Discover benchmarks
    if include_benchmarks is not None:
        # Whitelist mode: only aggregate benchmarks that were actually run in
        # this invocation. Directory names may differ from canonical names
        # (e.g. evalscope writes the hle benchmark to the hle_low directory),
        # so we canonicalize each directory name via BENCHMARK_NAME_ALIAS and
        # check membership in the caller's list. Order follows the caller's
        # include_benchmarks list.
        include_set = set(include_benchmarks)
        discovered = []  # list of (dir_name, canonical_name)
        for bench_dir in output_dir.iterdir():
            if not bench_dir.is_dir() or bench_dir.name == safe_model:
                continue
            if not find_all_reports(output_dir, bench_dir.name, model_name):
                continue
            canonical = BENCHMARK_NAME_ALIAS.get(bench_dir.name, bench_dir.name)
            if canonical in include_set or bench_dir.name in include_set:
                discovered.append((bench_dir.name, canonical))

        order = {name: idx for idx, name in enumerate(include_benchmarks)}
        discovered.sort(key=lambda x: order.get(x[1], len(include_benchmarks)))

        benchmarks = [dir_name for dir_name, _ in discovered]
    else:
        # Legacy behaviour: scan everything on disk.
        benchmarks = []
        for bench_dir in sorted(output_dir.iterdir()):
            if not bench_dir.is_dir() or bench_dir.name == safe_model:
                continue
            reports = find_all_reports(output_dir, bench_dir.name, model_name)
            if reports:
                benchmarks.append(bench_dir.name)

    rows = []
    for benchmark in benchmarks:
        row = collect_benchmark(output_dir, benchmark, model_name)
        if row:
            rows.append(row)

    if not rows:
        print(f'No results found for model {model_name} under {output_dir}')
        return None, None

    df = pd.DataFrame(rows, columns=OUTPUT_COLUMNS)
    df = df.drop_duplicates(subset=['Benchmark'], keep='first')

    # Add total row
    total_score = df['得分'].mean()
    total_time = df['实测时间(h)'].sum()
    total_row = {
        '分类': '总计',
        'Benchmark': '',
        '得分': round(total_score, 4),
        '实测时间(h)': round(total_time, 4),
    }
    for col in OUTPUT_COLUMNS:
        if col not in total_row:
            total_row[col] = np.nan
    df = pd.concat([df, pd.DataFrame([total_row], columns=OUTPUT_COLUMNS)], ignore_index=True)

    summary_dir = output_dir / 'results' / safe_model
    summary_dir.mkdir(parents=True, exist_ok=True)

    if out_name is None:
        out_name = safe_model

    csv_path = summary_dir / f'{out_name}.csv'

    # Excel can optionally be written to a separate project-level results dir
    # so that the latest summary per model is easy to find independently of
    # the per-run output directory.
    if excel_output_dir is not None:
        xlsx_dir = Path(excel_output_dir)
        xlsx_dir.mkdir(parents=True, exist_ok=True)
        xlsx_path = xlsx_dir / f'{out_name}.xlsx'
    else:
        xlsx_path = summary_dir / f'{out_name}.xlsx'

    df.to_csv(csv_path, index=False, encoding='utf-8-sig')
    df.to_excel(xlsx_path, index=False)

    print(f'Summary written to:')
    print(f'  CSV:  {csv_path}')
    print(f'  Excel: {xlsx_path}')
    return csv_path, xlsx_path


def main():
    parser = argparse.ArgumentParser(description='Collect EvalScope benchmark results into Excel/CSV')
    parser.add_argument('--output-dir', required=True, help='EvalScope output directory')
    parser.add_argument('--model', default='DeepSeek-V4-Flash-Int8', help='Model name used in reports')
    parser.add_argument('--out-name', default=None, help='Output file name (without extension); defaults to safe model name')
    parser.add_argument('--include', default=None,
                        help='Comma-separated benchmark whitelist (default: all on disk)')
    parser.add_argument('--benchmarks', '--benchmarks', dest='benchmarks', default=None,
                        help='Alias for --include')
    parser.add_argument('--excel-output-dir', default=None,
                        help='Separate directory for the Excel summary; defaults to <project_root>/results')
    args = parser.parse_args()

    # Default Excel output dir to project-level results/ so the latest summary
    # per model is easy to find independently of the per-run output directory.
    if args.excel_output_dir is None:
        excel_output_dir = Path(__file__).parent.parent / 'results'
    else:
        excel_output_dir = Path(args.excel_output_dir)

    include_benchmarks = None
    if args.include:
        include_benchmarks = [b.strip() for b in args.include.split(',') if b.strip()]
    elif args.benchmarks:
        include_benchmarks = [b.strip() for b in args.benchmarks.split(',') if b.strip()]

    if include_benchmarks is not None:
        eval_benchmark(include_benchmarks, Path(args.output_dir), args.model, args.out_name,
                        excel_output_dir=excel_output_dir)
    else:
        collect_all(Path(args.output_dir), args.model, args.out_name,
                    excel_output_dir=excel_output_dir)


if __name__ == '__main__':
    main()