From 8b0a413cf17c879a831daf6e4c65c289e7f00b8d Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 3 Sep 2026 04:11:23 +0000 Subject: [PATCH] Drop Harbor trial wall-clock from summary perf metrics. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent CSV rows now use jsonl per-call stats or the report request summary only, so TTFT/latency stay on the same request口径. Co-authored-by: Cursor --- bash/collect_results.py | 136 +++++------------------------- bash/tests/test_summary_upsert.py | 53 ++++++++++++ 2 files changed, 73 insertions(+), 116 deletions(-) diff --git a/bash/collect_results.py b/bash/collect_results.py index 636d347..7045c29 100644 --- a/bash/collect_results.py +++ b/bash/collect_results.py @@ -200,69 +200,6 @@ def read_predictions_with_index(pred_file: Path): yield {'index': obj.get('index'), 'perf_metrics': pm} -def read_agent_perf_from_trajectory(pred_file: Path): - """Extract approximate perf metrics from agent trajectory files. - - Agent/sandbox benchmarks (e.g. terminal_bench_v2_1) do not record per-call - latency/TTFT/TPOT through EvalScope's model wrapper. However, the trial - trajectory contains step timestamps and final token counts. This function - yields synthetic ``{'index', 'perf_metrics'}`` records with: - - - ``latency``: wall-clock trial duration in seconds - - ``input_tokens``: total prompt tokens from the agent run - - ``output_tokens``: total completion tokens from the agent run - - ``ttft`` / ``tpot``: not available, left as None - """ - if not pred_file.exists(): - return - from datetime import datetime - with open(pred_file, 'r', encoding='utf-8') as f: - for line in f: - line = line.strip() - if not line: - continue - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - idx = obj.get('index') - model_output = obj.get('model_output', {}) - content = '' - choices = model_output.get('choices', []) - if choices and 'message' in choices[0]: - content = choices[0]['message'].get('content', '') - if not content or not isinstance(content, str) or not content.startswith('file://'): - continue - traj_path = Path(content[7:]) / 'agent' / 'trajectory.json' - if not traj_path.exists(): - continue - try: - traj = json.loads(traj_path.read_text(encoding='utf-8')) - except Exception: - continue - steps = traj.get('steps', []) - duration = None - if len(steps) >= 2: - try: - first_ts = datetime.fromisoformat(steps[0]['timestamp']) - last_ts = datetime.fromisoformat(steps[-1]['timestamp']) - duration = (last_ts - first_ts).total_seconds() - except Exception: - pass - final_metrics = traj.get('final_metrics', {}) - prompt_tokens = final_metrics.get('total_prompt_tokens') - completion_tokens = final_metrics.get('total_completion_tokens') - pm = {} - if duration is not None: - pm['latency'] = duration - if prompt_tokens is not None: - pm['input_tokens'] = int(prompt_tokens) - if completion_tokens is not None: - pm['output_tokens'] = int(completion_tokens) - if pm: - yield {'index': idx, 'perf_metrics': pm} - - def load_backup_summary(output_dir: Path, benchmark: str, model_name: str): """Load the durable ``perf_stats_backup/__.json``. @@ -607,59 +544,26 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str): tpot_p99, ) = _assign_request_perf(chosen) else: - # Last resort: Harbor trajectory wall-clock (no TTFT/TPOT). - if pred_files: - for obj in read_agent_perf_from_trajectory(pred_files[0]): - idx = obj['index'] - key = ('trajectory', idx) - if idx is not None: - if key in seen_keys: - continue - seen_keys.add(key) - sample_indexes.append(idx) - pm = obj['perf_metrics'] - if pm.get('latency') is not None: - latencies.append(float(pm['latency'])) - if pm.get('input_tokens') is not None: - input_tokens.append(int(pm['input_tokens'])) - if pm.get('output_tokens') is not None: - output_tokens.append(int(pm['output_tokens'])) - if latencies: - latency_mean = float(np.mean(latencies)) - total_compute_time = float(np.sum(latencies)) - total_output_tokens = sum(output_tokens) - avg_output_tps = total_output_tokens / total_compute_time if total_compute_time > 0 else np.nan - avg_req_ps = len(latencies) / total_compute_time if total_compute_time > 0 else np.nan - input_tok_mean = float(np.mean(input_tokens)) if input_tokens else np.nan - output_tok_mean = float(np.mean(output_tokens)) if output_tokens else np.nan - total_tokens = sum(input_tokens) + sum(output_tokens) - ttft_mean = np.nan - ttft_p90 = np.nan - ttft_p99 = np.nan - tpot_mean = np.nan - tpot_p90 = np.nan - tpot_p99 = np.nan - if n_samples_unique < len(latencies): - n_samples_unique = len(latencies) - else: - latency_mean = np.nan - avg_output_tps = np.nan - avg_req_ps = np.nan - input_tok_mean = np.nan - output_tok_mean = np.nan - total_tokens = np.nan - ttft_mean = np.nan - ttft_p90 = np.nan - ttft_p99 = np.nan - tpot_mean = np.nan - tpot_p90 = np.nan - tpot_p99 = np.nan - if not n_samples_unique and reports: - try: - data = json.loads(reports[0].read_text(encoding='utf-8')) - n_samples_unique = data.get('num', 0) - except Exception: - pass + # No per-call jsonl metrics and no report/backup request summary. + # Do not infer latency from Harbor trial wall-clock. + latency_mean = np.nan + avg_output_tps = np.nan + avg_req_ps = np.nan + input_tok_mean = np.nan + output_tok_mean = np.nan + total_tokens = np.nan + ttft_mean = np.nan + ttft_p90 = np.nan + ttft_p99 = np.nan + tpot_mean = np.nan + tpot_p90 = np.nan + tpot_p99 = np.nan + if not n_samples_unique and reports: + try: + data = json.loads(reports[0].read_text(encoding='utf-8')) + n_samples_unique = data.get('num', 0) + except Exception: + pass # Duration: prefer the durable active timer maintained by perf_backup.py, # which only counts time when run_task() is actually executing. This avoids diff --git a/bash/tests/test_summary_upsert.py b/bash/tests/test_summary_upsert.py index 2718a89..d4beede 100644 --- a/bash/tests/test_summary_upsert.py +++ b/bash/tests/test_summary_upsert.py @@ -157,3 +157,56 @@ def test_agent_bench_uses_report_per_request_perf(tmp_path: Path): assert row['累计总tokens'] == 176681 assert row['输出TPS'] == 140.06 + +def test_agent_jsonl_does_not_use_trial_wall_clock(tmp_path: Path): + import json + + output_dir = tmp_path / 'output' + bench = 'terminal_bench_v2_1' + reports = output_dir / bench / 'seed_42' / 'reports' + preds = output_dir / bench / 'seed_42' / 'predictions' + trial = output_dir / bench / 'seed_42' / 'trials' / 'task__abc' + reports.mkdir(parents=True) + preds.mkdir(parents=True) + (trial / 'agent').mkdir(parents=True) + (reports / f'{bench}.json').write_text( + json.dumps({ + 'score': 0.0, + 'num': 1, + 'metrics': [{ + 'identity': {'name': 'accuracy', 'aggregation': 'mean', 'dimensions': {}}, + 'score': 0.0, + 'num': 1, + }], + }), + encoding='utf-8', + ) + (trial / 'agent' / 'trajectory.json').write_text( + json.dumps({ + 'steps': [ + {'timestamp': '2026-09-03T00:00:00'}, + {'timestamp': '2026-09-03T00:11:46'}, + ], + 'final_metrics': { + 'total_prompt_tokens': 638286, + 'total_completion_tokens': 154502, + }, + }), + encoding='utf-8', + ) + (preds / f'{bench}__m.jsonl').write_text( + json.dumps({ + 'index': 0, + 'model_output': { + 'choices': [{'message': {'content': f'file://{trial}'}}], + }, + }) + '\n', + encoding='utf-8', + ) + + row = collect_benchmark(output_dir, bench, 'm') + assert row['总样本数'] == 1 + assert np.isnan(row['延迟_mean(s)']) + assert np.isnan(row['TTFT_mean(s)']) + assert np.isnan(row['累计总tokens']) +