13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
126 lines
4.5 KiB
Python
126 lines
4.5 KiB
Python
# Copyright (c) Alibaba, Inc. and its affiliates.
|
|
"""Thread-safe HTTP request success tracking for vendor API evaluation.
|
|
|
|
Counts every real HTTP attempt (including retries). ``context_length`` /
|
|
``content_filter`` responses are tracked as ``client_errors`` and are excluded
|
|
from the vendor ``success_rate`` denominator.
|
|
|
|
``success_rate`` = success_attempts / (success_attempts + failed_attempts)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
from collections import Counter
|
|
from typing import Any, Dict, Optional # BaseException is a builtin
|
|
|
|
# OpenAI-style error codes treated as client/content issues, not vendor outages.
|
|
_CLIENT_ERROR_CODES = frozenset({
|
|
'context_length_exceeded',
|
|
'invalid_prompt',
|
|
'content_policy_violation',
|
|
'content_filter',
|
|
})
|
|
|
|
# Substrings in error messages (OpenAI-compatible + Anthropic + DashScope, etc.).
|
|
_CLIENT_ERROR_MSG_PATTERNS = (
|
|
'context_length',
|
|
'input length',
|
|
'maximum context length',
|
|
'context window',
|
|
'too many tokens',
|
|
'prompt is too long',
|
|
'input is too long',
|
|
'exceed context limit',
|
|
'content_policy',
|
|
'content_filter',
|
|
'content filtering',
|
|
'content filter',
|
|
)
|
|
|
|
|
|
def is_client_error(exc: Optional[BaseException]) -> bool:
|
|
"""True for context-length / content-filter HTTP errors.
|
|
|
|
These are recorded as ``client_errors`` and excluded from vendor
|
|
``success_rate``.
|
|
"""
|
|
if exc is None:
|
|
return False
|
|
code = getattr(exc, 'code', None)
|
|
if isinstance(code, str) and code in _CLIENT_ERROR_CODES:
|
|
return True
|
|
parts = [str(exc)]
|
|
message = getattr(exc, 'message', None)
|
|
if message:
|
|
parts.append(str(message))
|
|
body = getattr(exc, 'body', None)
|
|
if isinstance(body, dict):
|
|
nested = body.get('message') or body.get('error')
|
|
if nested is not None:
|
|
parts.append(str(nested))
|
|
blob = ' '.join(parts).lower()
|
|
return any(p in blob for p in _CLIENT_ERROR_MSG_PATTERNS)
|
|
|
|
|
|
class RequestStats:
|
|
"""Accumulate per-HTTP-attempt and per-logical-generate() outcomes."""
|
|
|
|
def __init__(self) -> None:
|
|
self._lock = threading.Lock()
|
|
self.success_attempts = 0
|
|
self.failed_attempts = 0
|
|
self.client_errors = 0
|
|
self.logical_calls = 0
|
|
self.logical_success = 0
|
|
self.error_types: Counter[str] = Counter()
|
|
self.client_error_types: Counter[str] = Counter()
|
|
|
|
@property
|
|
def total_attempts(self) -> int:
|
|
return self.success_attempts + self.failed_attempts + self.client_errors
|
|
|
|
def on_attempt(self, ok: bool, exc: Optional[BaseException] = None) -> None:
|
|
"""Callback for ``retry_call`` / ``async_retry_call`` (one HTTP try)."""
|
|
with self._lock:
|
|
if ok:
|
|
self.success_attempts += 1
|
|
return
|
|
name = type(exc).__name__ if exc is not None else 'UnknownError'
|
|
if is_client_error(exc):
|
|
self.client_errors += 1
|
|
self.client_error_types[name] += 1
|
|
else:
|
|
self.failed_attempts += 1
|
|
self.error_types[name] += 1
|
|
|
|
def record_logical(self, ok: bool) -> None:
|
|
"""One ``generate()`` / ``generate_async()`` invocation after retries."""
|
|
with self._lock:
|
|
self.logical_calls += 1
|
|
if ok:
|
|
self.logical_success += 1
|
|
|
|
def snapshot(self) -> Dict[str, Any]:
|
|
"""Serialize for ``perf_metrics.summary.request``."""
|
|
with self._lock:
|
|
success = self.success_attempts
|
|
failed = self.failed_attempts
|
|
client = self.client_errors
|
|
vendor_denom = success + failed
|
|
success_rate = (success / vendor_denom) if vendor_denom > 0 else None
|
|
logical_calls = self.logical_calls
|
|
logical_success = self.logical_success
|
|
logical_rate = (logical_success / logical_calls) if logical_calls > 0 else None
|
|
return {
|
|
'total_attempts': success + failed + client,
|
|
'success_attempts': success,
|
|
'failed_attempts': failed,
|
|
'client_errors': client,
|
|
'success_rate': None if success_rate is None else round(success_rate, 4),
|
|
'logical_calls': logical_calls,
|
|
'logical_success': logical_success,
|
|
'logical_success_rate': None if logical_rate is None else round(logical_rate, 4),
|
|
'error_types': dict(self.error_types),
|
|
'client_error_types': dict(self.client_error_types),
|
|
}
|