Drop Harbor trial wall-clock from summary perf metrics.

Agent CSV rows now use jsonl per-call stats or the report request summary only, so TTFT/latency stay on the same request口径.

Co-authored-by: Cursor <cursoragent@cursor.com>

bash/collect_results.py

@@ -200,69 +200,6 @@ def read_predictions_with_index(pred_file: Path):
             yield {'index': obj.get('index'), 'perf_metrics': pm}
 
 
-def read_agent_perf_from_trajectory(pred_file: Path):
-    """Extract approximate perf metrics from agent trajectory files.
-
-    Agent/sandbox benchmarks (e.g. terminal_bench_v2_1) do not record per-call
-    latency/TTFT/TPOT through EvalScope's model wrapper. However, the trial
-    trajectory contains step timestamps and final token counts. This function
-    yields synthetic ``{'index', 'perf_metrics'}`` records with:
-
-      - ``latency``: wall-clock trial duration in seconds
-      - ``input_tokens``: total prompt tokens from the agent run
-      - ``output_tokens``: total completion tokens from the agent run
-      - ``ttft`` / ``tpot``: not available, left as None
-    """
-    if not pred_file.exists():
-        return
-    from datetime import datetime
-    with open(pred_file, 'r', encoding='utf-8') as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                obj = json.loads(line)
-            except json.JSONDecodeError:
-                continue
-            idx = obj.get('index')
-            model_output = obj.get('model_output', {})
-            content = ''
-            choices = model_output.get('choices', [])
-            if choices and 'message' in choices[0]:
-                content = choices[0]['message'].get('content', '')
-            if not content or not isinstance(content, str) or not content.startswith('file://'):
-                continue
-            traj_path = Path(content[7:]) / 'agent' / 'trajectory.json'
-            if not traj_path.exists():
-                continue
-            try:
-                traj = json.loads(traj_path.read_text(encoding='utf-8'))
-            except Exception:
-                continue
-            steps = traj.get('steps', [])
-            duration = None
-            if len(steps) >= 2:
-                try:
-                    first_ts = datetime.fromisoformat(steps[0]['timestamp'])
-                    last_ts = datetime.fromisoformat(steps[-1]['timestamp'])
-                    duration = (last_ts - first_ts).total_seconds()
-                except Exception:
-                    pass
-            final_metrics = traj.get('final_metrics', {})
-            prompt_tokens = final_metrics.get('total_prompt_tokens')
-            completion_tokens = final_metrics.get('total_completion_tokens')
-            pm = {}
-            if duration is not None:
-                pm['latency'] = duration
-            if prompt_tokens is not None:
-                pm['input_tokens'] = int(prompt_tokens)
-            if completion_tokens is not None:
-                pm['output_tokens'] = int(completion_tokens)
-            if pm:
-                yield {'index': idx, 'perf_metrics': pm}
-
-
 def load_backup_summary(output_dir: Path, benchmark: str, model_name: str):
     """Load the durable ``perf_stats_backup/<benchmark>__<model>.json``.
 
@@ -607,59 +544,26 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
             tpot_p99,
         ) = _assign_request_perf(chosen)
     else:
-        # Last resort: Harbor trajectory wall-clock (no TTFT/TPOT).
+        # No per-call jsonl metrics and no report/backup request summary.
-        if pred_files:
+        # Do not infer latency from Harbor trial wall-clock.
-            for obj in read_agent_perf_from_trajectory(pred_files[0]):
+        latency_mean = np.nan
-                idx = obj['index']
+        avg_output_tps = np.nan
-                key = ('trajectory', idx)
+        avg_req_ps = np.nan
-                if idx is not None:
+        input_tok_mean = np.nan
-                    if key in seen_keys:
+        output_tok_mean = np.nan
-                        continue
+        total_tokens = np.nan
-                    seen_keys.add(key)
+        ttft_mean = np.nan
-                    sample_indexes.append(idx)
+        ttft_p90 = np.nan
-                pm = obj['perf_metrics']
+        ttft_p99 = np.nan
-                if pm.get('latency') is not None:
+        tpot_mean = np.nan
-                    latencies.append(float(pm['latency']))
+        tpot_p90 = np.nan
-                if pm.get('input_tokens') is not None:
+        tpot_p99 = np.nan
-                    input_tokens.append(int(pm['input_tokens']))
+        if not n_samples_unique and reports:
-                if pm.get('output_tokens') is not None:
+            try:
-                    output_tokens.append(int(pm['output_tokens']))
+                data = json.loads(reports[0].read_text(encoding='utf-8'))
-        if latencies:
+                n_samples_unique = data.get('num', 0)
-            latency_mean = float(np.mean(latencies))
+            except Exception:
-            total_compute_time = float(np.sum(latencies))
+                pass
-            total_output_tokens = sum(output_tokens)
-            avg_output_tps = total_output_tokens / total_compute_time if total_compute_time > 0 else np.nan
-            avg_req_ps = len(latencies) / total_compute_time if total_compute_time > 0 else np.nan
-            input_tok_mean = float(np.mean(input_tokens)) if input_tokens else np.nan
-            output_tok_mean = float(np.mean(output_tokens)) if output_tokens else np.nan
-            total_tokens = sum(input_tokens) + sum(output_tokens)
-            ttft_mean = np.nan
-            ttft_p90 = np.nan
-            ttft_p99 = np.nan
-            tpot_mean = np.nan
-            tpot_p90 = np.nan
-            tpot_p99 = np.nan
-            if n_samples_unique < len(latencies):
-                n_samples_unique = len(latencies)
-        else:
-            latency_mean = np.nan
-            avg_output_tps = np.nan
-            avg_req_ps = np.nan
-            input_tok_mean = np.nan
-            output_tok_mean = np.nan
-            total_tokens = np.nan
-            ttft_mean = np.nan
-            ttft_p90 = np.nan
-            ttft_p99 = np.nan
-            tpot_mean = np.nan
-            tpot_p90 = np.nan
-            tpot_p99 = np.nan
-            if not n_samples_unique and reports:
-                try:
-                    data = json.loads(reports[0].read_text(encoding='utf-8'))
-                    n_samples_unique = data.get('num', 0)
-                except Exception:
-                    pass
 
     # Duration: prefer the durable active timer maintained by perf_backup.py,
     # which only counts time when run_task() is actually executing. This avoids

bash/tests/test_summary_upsert.py

@@ -157,3 +157,56 @@ def test_agent_bench_uses_report_per_request_perf(tmp_path: Path):
     assert row['累计总tokens'] == 176681
     assert row['输出TPS'] == 140.06
 
+
+def test_agent_jsonl_does_not_use_trial_wall_clock(tmp_path: Path):
+    import json
+
+    output_dir = tmp_path / 'output'
+    bench = 'terminal_bench_v2_1'
+    reports = output_dir / bench / 'seed_42' / 'reports'
+    preds = output_dir / bench / 'seed_42' / 'predictions'
+    trial = output_dir / bench / 'seed_42' / 'trials' / 'task__abc'
+    reports.mkdir(parents=True)
+    preds.mkdir(parents=True)
+    (trial / 'agent').mkdir(parents=True)
+    (reports / f'{bench}.json').write_text(
+        json.dumps({
+            'score': 0.0,
+            'num': 1,
+            'metrics': [{
+                'identity': {'name': 'accuracy', 'aggregation': 'mean', 'dimensions': {}},
+                'score': 0.0,
+                'num': 1,
+            }],
+        }),
+        encoding='utf-8',
+    )
+    (trial / 'agent' / 'trajectory.json').write_text(
+        json.dumps({
+            'steps': [
+                {'timestamp': '2026-09-03T00:00:00'},
+                {'timestamp': '2026-09-03T00:11:46'},
+            ],
+            'final_metrics': {
+                'total_prompt_tokens': 638286,
+                'total_completion_tokens': 154502,
+            },
+        }),
+        encoding='utf-8',
+    )
+    (preds / f'{bench}__m.jsonl').write_text(
+        json.dumps({
+            'index': 0,
+            'model_output': {
+                'choices': [{'message': {'content': f'file://{trial}'}}],
+            },
+        }) + '\n',
+        encoding='utf-8',
+    )
+
+    row = collect_benchmark(output_dir, bench, 'm')
+    assert row['总样本数'] == 1
+    assert np.isnan(row['延迟_mean(s)'])
+    assert np.isnan(row['TTFT_mean(s)'])
+    assert np.isnan(row['累计总tokens'])
+