13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
39 lines
1.6 KiB
Diff
39 lines
1.6 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/evaluator/evaluator.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/evaluator/evaluator.py 2026-09-01 03:48:30.955904239 +0000
|
|
@@ -534,12 +534,15 @@
|
|
if self.task_config.collect_perf:
|
|
report.perf_metrics = self.perf_collector.get_perf_dict() or None
|
|
|
|
+ # Vendor HTTP request success rate (counted per attempt, including retries).
|
|
+ self._inject_request_stats(report)
|
|
+
|
|
# Save the complete report to file
|
|
report.to_json(report_file)
|
|
logger.info(f'Dump report to: {report_file} \n')
|
|
|
|
# Print per-benchmark perf table when perf data is available
|
|
- if self.task_config.collect_perf and report.perf_metrics:
|
|
+ if report.perf_metrics:
|
|
try:
|
|
perf_table = gen_perf_table(report_list=[report])
|
|
if perf_table:
|
|
@@ -549,6 +552,18 @@
|
|
|
|
return report
|
|
|
|
+ def _inject_request_stats(self, report: Report) -> None:
|
|
+ """Attach HTTP request success stats under ``perf_metrics.summary.request``."""
|
|
+ api = getattr(self.model, 'api', None)
|
|
+ stats = getattr(api, 'request_stats', None)
|
|
+ if stats is None or stats.total_attempts <= 0:
|
|
+ return
|
|
+ payload = stats.snapshot()
|
|
+ if report.perf_metrics is None:
|
|
+ report.perf_metrics = {}
|
|
+ summary = report.perf_metrics.setdefault('summary', {})
|
|
+ summary['request'] = payload
|
|
+
|
|
def finalize(self, *args, **kwargs):
|
|
self.benchmark.finalize(*args, **kwargs)
|
|
self.cache_manager.close()
|