Drop Harbor trial wall-clock from summary perf metrics.
Agent CSV rows now use jsonl per-call stats or the report request summary only, so TTFT/latency stay on the same request口径. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
sora
parent
44303d8953
commit
8b0a413cf1
@ -200,69 +200,6 @@ def read_predictions_with_index(pred_file: Path):
|
||||
yield {'index': obj.get('index'), 'perf_metrics': pm}
|
||||
|
||||
|
||||
def read_agent_perf_from_trajectory(pred_file: Path):
|
||||
"""Extract approximate perf metrics from agent trajectory files.
|
||||
|
||||
Agent/sandbox benchmarks (e.g. terminal_bench_v2_1) do not record per-call
|
||||
latency/TTFT/TPOT through EvalScope's model wrapper. However, the trial
|
||||
trajectory contains step timestamps and final token counts. This function
|
||||
yields synthetic ``{'index', 'perf_metrics'}`` records with:
|
||||
|
||||
- ``latency``: wall-clock trial duration in seconds
|
||||
- ``input_tokens``: total prompt tokens from the agent run
|
||||
- ``output_tokens``: total completion tokens from the agent run
|
||||
- ``ttft`` / ``tpot``: not available, left as None
|
||||
"""
|
||||
if not pred_file.exists():
|
||||
return
|
||||
from datetime import datetime
|
||||
with open(pred_file, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
idx = obj.get('index')
|
||||
model_output = obj.get('model_output', {})
|
||||
content = ''
|
||||
choices = model_output.get('choices', [])
|
||||
if choices and 'message' in choices[0]:
|
||||
content = choices[0]['message'].get('content', '')
|
||||
if not content or not isinstance(content, str) or not content.startswith('file://'):
|
||||
continue
|
||||
traj_path = Path(content[7:]) / 'agent' / 'trajectory.json'
|
||||
if not traj_path.exists():
|
||||
continue
|
||||
try:
|
||||
traj = json.loads(traj_path.read_text(encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
steps = traj.get('steps', [])
|
||||
duration = None
|
||||
if len(steps) >= 2:
|
||||
try:
|
||||
first_ts = datetime.fromisoformat(steps[0]['timestamp'])
|
||||
last_ts = datetime.fromisoformat(steps[-1]['timestamp'])
|
||||
duration = (last_ts - first_ts).total_seconds()
|
||||
except Exception:
|
||||
pass
|
||||
final_metrics = traj.get('final_metrics', {})
|
||||
prompt_tokens = final_metrics.get('total_prompt_tokens')
|
||||
completion_tokens = final_metrics.get('total_completion_tokens')
|
||||
pm = {}
|
||||
if duration is not None:
|
||||
pm['latency'] = duration
|
||||
if prompt_tokens is not None:
|
||||
pm['input_tokens'] = int(prompt_tokens)
|
||||
if completion_tokens is not None:
|
||||
pm['output_tokens'] = int(completion_tokens)
|
||||
if pm:
|
||||
yield {'index': idx, 'perf_metrics': pm}
|
||||
|
||||
|
||||
def load_backup_summary(output_dir: Path, benchmark: str, model_name: str):
|
||||
"""Load the durable ``perf_stats_backup/<benchmark>__<model>.json``.
|
||||
|
||||
@ -607,59 +544,26 @@ def collect_benchmark(output_dir: Path, benchmark: str, model_name: str):
|
||||
tpot_p99,
|
||||
) = _assign_request_perf(chosen)
|
||||
else:
|
||||
# Last resort: Harbor trajectory wall-clock (no TTFT/TPOT).
|
||||
if pred_files:
|
||||
for obj in read_agent_perf_from_trajectory(pred_files[0]):
|
||||
idx = obj['index']
|
||||
key = ('trajectory', idx)
|
||||
if idx is not None:
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
sample_indexes.append(idx)
|
||||
pm = obj['perf_metrics']
|
||||
if pm.get('latency') is not None:
|
||||
latencies.append(float(pm['latency']))
|
||||
if pm.get('input_tokens') is not None:
|
||||
input_tokens.append(int(pm['input_tokens']))
|
||||
if pm.get('output_tokens') is not None:
|
||||
output_tokens.append(int(pm['output_tokens']))
|
||||
if latencies:
|
||||
latency_mean = float(np.mean(latencies))
|
||||
total_compute_time = float(np.sum(latencies))
|
||||
total_output_tokens = sum(output_tokens)
|
||||
avg_output_tps = total_output_tokens / total_compute_time if total_compute_time > 0 else np.nan
|
||||
avg_req_ps = len(latencies) / total_compute_time if total_compute_time > 0 else np.nan
|
||||
input_tok_mean = float(np.mean(input_tokens)) if input_tokens else np.nan
|
||||
output_tok_mean = float(np.mean(output_tokens)) if output_tokens else np.nan
|
||||
total_tokens = sum(input_tokens) + sum(output_tokens)
|
||||
ttft_mean = np.nan
|
||||
ttft_p90 = np.nan
|
||||
ttft_p99 = np.nan
|
||||
tpot_mean = np.nan
|
||||
tpot_p90 = np.nan
|
||||
tpot_p99 = np.nan
|
||||
if n_samples_unique < len(latencies):
|
||||
n_samples_unique = len(latencies)
|
||||
else:
|
||||
latency_mean = np.nan
|
||||
avg_output_tps = np.nan
|
||||
avg_req_ps = np.nan
|
||||
input_tok_mean = np.nan
|
||||
output_tok_mean = np.nan
|
||||
total_tokens = np.nan
|
||||
ttft_mean = np.nan
|
||||
ttft_p90 = np.nan
|
||||
ttft_p99 = np.nan
|
||||
tpot_mean = np.nan
|
||||
tpot_p90 = np.nan
|
||||
tpot_p99 = np.nan
|
||||
if not n_samples_unique and reports:
|
||||
try:
|
||||
data = json.loads(reports[0].read_text(encoding='utf-8'))
|
||||
n_samples_unique = data.get('num', 0)
|
||||
except Exception:
|
||||
pass
|
||||
# No per-call jsonl metrics and no report/backup request summary.
|
||||
# Do not infer latency from Harbor trial wall-clock.
|
||||
latency_mean = np.nan
|
||||
avg_output_tps = np.nan
|
||||
avg_req_ps = np.nan
|
||||
input_tok_mean = np.nan
|
||||
output_tok_mean = np.nan
|
||||
total_tokens = np.nan
|
||||
ttft_mean = np.nan
|
||||
ttft_p90 = np.nan
|
||||
ttft_p99 = np.nan
|
||||
tpot_mean = np.nan
|
||||
tpot_p90 = np.nan
|
||||
tpot_p99 = np.nan
|
||||
if not n_samples_unique and reports:
|
||||
try:
|
||||
data = json.loads(reports[0].read_text(encoding='utf-8'))
|
||||
n_samples_unique = data.get('num', 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Duration: prefer the durable active timer maintained by perf_backup.py,
|
||||
# which only counts time when run_task() is actually executing. This avoids
|
||||
|
||||
@ -157,3 +157,56 @@ def test_agent_bench_uses_report_per_request_perf(tmp_path: Path):
|
||||
assert row['累计总tokens'] == 176681
|
||||
assert row['输出TPS'] == 140.06
|
||||
|
||||
|
||||
def test_agent_jsonl_does_not_use_trial_wall_clock(tmp_path: Path):
|
||||
import json
|
||||
|
||||
output_dir = tmp_path / 'output'
|
||||
bench = 'terminal_bench_v2_1'
|
||||
reports = output_dir / bench / 'seed_42' / 'reports'
|
||||
preds = output_dir / bench / 'seed_42' / 'predictions'
|
||||
trial = output_dir / bench / 'seed_42' / 'trials' / 'task__abc'
|
||||
reports.mkdir(parents=True)
|
||||
preds.mkdir(parents=True)
|
||||
(trial / 'agent').mkdir(parents=True)
|
||||
(reports / f'{bench}.json').write_text(
|
||||
json.dumps({
|
||||
'score': 0.0,
|
||||
'num': 1,
|
||||
'metrics': [{
|
||||
'identity': {'name': 'accuracy', 'aggregation': 'mean', 'dimensions': {}},
|
||||
'score': 0.0,
|
||||
'num': 1,
|
||||
}],
|
||||
}),
|
||||
encoding='utf-8',
|
||||
)
|
||||
(trial / 'agent' / 'trajectory.json').write_text(
|
||||
json.dumps({
|
||||
'steps': [
|
||||
{'timestamp': '2026-09-03T00:00:00'},
|
||||
{'timestamp': '2026-09-03T00:11:46'},
|
||||
],
|
||||
'final_metrics': {
|
||||
'total_prompt_tokens': 638286,
|
||||
'total_completion_tokens': 154502,
|
||||
},
|
||||
}),
|
||||
encoding='utf-8',
|
||||
)
|
||||
(preds / f'{bench}__m.jsonl').write_text(
|
||||
json.dumps({
|
||||
'index': 0,
|
||||
'model_output': {
|
||||
'choices': [{'message': {'content': f'file://{trial}'}}],
|
||||
},
|
||||
}) + '\n',
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
row = collect_benchmark(output_dir, bench, 'm')
|
||||
assert row['总样本数'] == 1
|
||||
assert np.isnan(row['延迟_mean(s)'])
|
||||
assert np.isnan(row['TTFT_mean(s)'])
|
||||
assert np.isnan(row['累计总tokens'])
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user