From 1e72a553ec9c5578cc516ba8606642a56cc44626 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Tue, 15 Sep 2026 10:06:51 +0000 Subject: [PATCH] Benchmark categories in results (English) Five categories (Code & Engineering / Reasoning & Math / Knowledge & Language / Long Context / Agents & Tools) now annotate the console summary table (new column), summary.csv (new field), and each report's run_info.category. Co-Authored-By: Claude --- evalharness/cli.py | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index ed8fb81..a45d06d 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -163,6 +163,30 @@ def _rich_console(): return None +BENCH_CATEGORIES = { + 'bigcodebench': 'Code & Engineering', 'humaneval': 'Code & Engineering', + 'live_code_bench': 'Code & Engineering', + 'swe_bench_verified': 'Code & Engineering', + 'aime24': 'Reasoning & Math', 'aime25': 'Reasoning & Math', + 'aime26': 'Reasoning & Math', 'hmmt26': 'Reasoning & Math', + 'imo_answerbench': 'Reasoning & Math', 'hle': 'Reasoning & Math', + 'gsm8k': 'Reasoning & Math', 'competition_math': 'Reasoning & Math', + 'bbh': 'Reasoning & Math', 'drop': 'Reasoning & Math', + 'gpqa_diamond': 'Knowledge & Language', 'mmlu_pro': 'Knowledge & Language', + 'simple_qa': 'Knowledge & Language', 'mmlu': 'Knowledge & Language', + 'cmmlu': 'Knowledge & Language', 'arc': 'Knowledge & Language', + 'hellaswag': 'Knowledge & Language', 'trivia_qa': 'Knowledge & Language', + 'winogrande': 'Knowledge & Language', + 'longbench_v2': 'Long Context', 'openai_mrcr': 'Long Context', + 'tau2_bench': 'Agents & Tools', 'general_fc': 'Agents & Tools', + 'bfcl_v3': 'Agents & Tools', +} + + +def bench_category(name: str) -> str: + return BENCH_CATEGORIES.get(name, '') + + def _load_bench_cfg(args, name: str) -> dict: """Merged YAML config for one bench: {default 段, bench 段}. @@ -664,7 +688,7 @@ def _cmd_eval_run(args) -> int: with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f: w = _csv.writer(f) - w.writerow(['benchmark', 'score', 'metric', 'num_samples', + w.writerow(['benchmark', 'category', 'score', 'metric', 'num_samples', 'time_h', 'time_s', 'extract_fail', 'success_rate', 'latency_mean_s', 'output_tps', 'request_qps', 'input_tokens_mean', 'output_tokens_mean', 'total_tokens', @@ -678,7 +702,8 @@ def _cmd_eval_run(args) -> int: if gname != 'perf' for g, v in (gv or {}).items() if isinstance(v, (int, float)))[:2000] - w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''), + w.writerow([r['name'], r.get('category', ''), _f3(r.get('value')), + r['metric'], r.get('n', ''), r.get('hours', ''), r.get('secs', ''), r.get('extract_fail', 0)] + [perf.get(k, '') for k in ( @@ -943,7 +968,9 @@ def _cmd_eval_run(args) -> int: # (days old, slower setup) -- that once reported 15.9h for a # one-hour run _wall = round(_time.time() - t0, 1) - rows.append({'name': name, 'metric': primary, + _cat = bench_category(name) + report.metric_groups.setdefault('run_info', {})['category'] = _cat + rows.append({'name': name, 'metric': primary, 'category': _cat, 'cached': (report.metric_groups.get('run_info', {}) .get('gen_fresh') == 0) if _repeats <= 1 else False, 'value': report.metrics.get(primary), @@ -971,6 +998,7 @@ def _cmd_eval_run(args) -> int: 'done', _time.time() - t0) except Exception as e: rows.append({'name': name, 'metric': '-', 'value': None, + 'category': bench_category(name), 'secs': round(_time.time() - t0, 1), 'ok': False, 'err': f'{type(e).__name__}: {str(e)[:100]}'}) print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr) @@ -1040,7 +1068,8 @@ def _print_summary_table(console, rows): t = Table(title='Run Summary', header_style='bold cyan', title_style='bold', expand=False) - for col, just in (('benchmark', 'left'), ('metric', 'left'), + for col, just in (('benchmark', 'left'), ('category', 'left'), + ('metric', 'left'), ('score', 'right'), ('n', 'right'), ('time', 'right'), ('tok in', 'right'), ('tok out', 'right'), ('in/s', 'right'), ('out/s', 'right')): @@ -1055,7 +1084,8 @@ def _print_summary_table(console, rows): tout = f'{to:,}' if to else '—' tis = f'{ti / wall:.0f}' if (wall > 1 and ti) else '—' tos = f'{to / wall:.0f}' if (wall > 1 and to) else '—' - t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, + t.add_row(r['name'], r.get('category', ''), r['metric'], v, + str(r.get('n', '')), tm, tin, tout, tis, tos, style='green' if r['ok'] else 'red') wall_all = sum(r.get('wall') or r.get('secs') or 0 for r in rows) @@ -1066,7 +1096,7 @@ def _print_summary_table(console, rows): t.add_section() tis_all = f'{ti_all / wall_all:.0f}' if wall_all > 1 else '' tos_all = f'{to_all / wall_all:.0f}' if wall_all > 1 else '' - t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', + t.add_row(f'[bold]{len(rows)} benchmarks[/bold]', '', '', f'{sum(1 for r in rows if r["ok"])}/{len(rows)} ok', str(n_all), tm_all, f'{ti_all:,}', f'{to_all:,}', tis_all, tos_all)