sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
evalstone/patches/local-uncommitted-evalscope.patch
sora 13274243a0 Bump vendored EvalScope and add K3-ready DPV4 configs. 
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-02 07:30:48 +00:00

896 lines
37 KiB
Diff
Raw Blame History

diff --git a/evalscope/evalscope/api/messages/__init__.py b/evalscope/evalscope/api/messages/__init__.py
index c79fac4..b606da7 100644
--- a/evalscope/evalscope/api/messages/__init__.py
+++ b/evalscope/evalscope/api/messages/__init__.py
@@ -11,4 +11,5 @@ from .chat_message import (
)
from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo
from .perf_metrics import PerformanceMetrics, PerfSummary
+from .request_stats import RequestStats, is_client_error
from .utils import parse_content_with_reasoning
diff --git a/evalscope/evalscope/api/model/model.py b/evalscope/evalscope/api/model/model.py
index b0231e8..b14cd0f 100644
--- a/evalscope/evalscope/api/model/model.py
+++ b/evalscope/evalscope/api/model/model.py
@@ -1,10 +1,12 @@
import abc
import asyncio
+from contextlib import contextmanager
from functools import partial
from pydantic_core import to_jsonable_python
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional, Sequence, Union
from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser
+from evalscope.api.messages.request_stats import RequestStats
from evalscope.api.registry import get_model_api
from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo
from evalscope.utils import get_logger, get_secret_value
@@ -43,6 +45,18 @@ class ModelAPI(abc.ABC):
self.base_url = base_url
self.api_key = api_key
self.config = config
+ self.request_stats = RequestStats()
+
+ @contextmanager
+ def _track_logical_request(self):
+ """Count one logical generate() call after HTTP retries have finished."""
+ try:
+ yield
+ except Exception:
+ self.request_stats.record_logical(ok=False)
+ raise
+ else:
+ self.request_stats.record_logical(ok=True)
@abc.abstractmethod
def generate(
diff --git a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py
index 63b5f73..bc11e03 100644
--- a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py
+++ b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py
@@ -51,6 +51,16 @@ COMMON_EXTRA_PARAMS = {
'description': 'Extra kwargs passed to Pier AgentConfig.kwargs.',
'value': {},
},
+ 'pier_model_prefix': {
+ 'type': 'str',
+ 'description': 'LiteLLM provider prefix prepended when TaskConfig.model has no slash (e.g. openai).',
+ 'value': 'openai',
+ },
+ 'pier_model_name': {
+ 'type': 'str',
+ 'description': 'Optional full Pier model name provider/model. Overrides TaskConfig.model when set.',
+ 'value': '',
+ },
}
@@ -65,6 +75,8 @@ class DeepSWEAdapter(AgentAdapter):
self.categories = self._as_list(extra_params.get('categories') or [])
self.sample_seed = extra_params.get('sample_seed')
self.pier_agent_kwargs = dict(extra_params.get('pier_agent_kwargs') or {})
+ self.pier_model_prefix = str(extra_params.get('pier_model_prefix') or 'openai')
+ self.pier_model_name = str(extra_params.get('pier_model_name') or '')
@staticmethod
def _as_list(value: Union[str, List[Any], Tuple[Any, ...]]) -> List[str]:
@@ -127,6 +139,8 @@ class DeepSWEAdapter(AgentAdapter):
from pier.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, VerifierConfig
task_id = str(sample.metadata['task_id'])
+ pier_kwargs = dict(self.pier_agent_kwargs)
+ pier_kwargs.setdefault('model_class', 'litellm')
config = JobConfig(
job_name=f'{task_id[:48].rstrip("_-")}__{uuid.uuid4().hex[:8]}',
@@ -140,9 +154,9 @@ class DeepSWEAdapter(AgentAdapter):
environment_build_timeout_multiplier=1.0,
agents=[AgentConfig(
name='mini-swe-agent',
- model_name=model.name,
- kwargs=self.pier_agent_kwargs,
- env={},
+ model_name=self._pier_model_name(model),
+ kwargs=pier_kwargs,
+ env=self._pier_agent_env(),
)],
environment=EnvironmentConfig(type='docker'),
verifier=VerifierConfig(env={}),
@@ -159,6 +173,38 @@ class DeepSWEAdapter(AgentAdapter):
self._raise_for_pier_failures(result_dict)
return result_dict
+ def _pier_model_name(self, model: Model) -> str:
+ name = self.pier_model_name or (model.name if model else '')
+ name = str(name).strip()
+ if name and '/' not in name:
+ name = f'{self.pier_model_prefix}/{name}'
+ return name
+
+ @staticmethod
+ def _plain_secret(value: Any) -> str:
+ if value is None:
+ return ''
+ if hasattr(value, 'get_secret_value'):
+ value = value.get_secret_value()
+ return str(value).strip()
+
+ def _pier_agent_env(self) -> Dict[str, str]:
+ env: Dict[str, str] = {
+ 'LITELLM_LOCAL_MODEL_COST_MAP': 'true',
+ }
+ tc = self._task_config
+ if tc is None:
+ return env
+ api_url = self._plain_secret(getattr(tc, 'api_url', None))
+ api_key = self._plain_secret(getattr(tc, 'api_key', None))
+ if api_url:
+ env['OPENAI_API_BASE'] = api_url
+ env['OPENAI_BASE_URL'] = api_url
+ if api_key and api_key != 'EMPTY':
+ env['OPENAI_API_KEY'] = api_key
+ env['MSWEA_API_KEY'] = api_key
+ return env
+
@staticmethod
def _raise_for_pier_failures(result_dict: Dict[str, Any]) -> None:
trial_results = result_dict.get('trial_results') or []
@@ -214,6 +260,8 @@ integrates it through Pier and runs each benchmark sample as one Pier Python API
- Dataset defaults to ModelScope `evalscope/deep-swe`
- DeepSWE runs through Pier's Docker environment in EvalScope
- Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API
+- If `TaskConfig.model` has no slash, EvalScope prefixes `pier_model_prefix` (default `openai/`) for Pier/LiteLLM
+- API base/key are forwarded as `OPENAI_API_BASE` / `OPENAI_API_KEY` from TaskConfig
""",
dataset_id=DEFAULT_MODELSCOPE_DATASET_ID,
eval_split='test',
diff --git a/evalscope/evalscope/evaluator/evaluator.py b/evalscope/evalscope/evaluator/evaluator.py
index 68a0987..2272503 100644
--- a/evalscope/evalscope/evaluator/evaluator.py
+++ b/evalscope/evalscope/evaluator/evaluator.py
@@ -534,12 +534,15 @@ class DefaultEvaluator(Evaluator):
if self.task_config.collect_perf:
report.perf_metrics = self.perf_collector.get_perf_dict() or None
+ # Vendor HTTP request success rate (counted per attempt, including retries).
+ self._inject_request_stats(report)
+
# Save the complete report to file
report.to_json(report_file)
logger.info(f'Dump report to: {report_file} \n')
# Print per-benchmark perf table when perf data is available
- if self.task_config.collect_perf and report.perf_metrics:
+ if report.perf_metrics:
try:
perf_table = gen_perf_table(report_list=[report])
if perf_table:
@@ -549,6 +552,18 @@ class DefaultEvaluator(Evaluator):
return report
+ def _inject_request_stats(self, report: Report) -> None:
+ """Attach HTTP request success stats under ``perf_metrics.summary.request``."""
+ api = getattr(self.model, 'api', None)
+ stats = getattr(api, 'request_stats', None)
+ if stats is None or stats.total_attempts <= 0:
+ return
+ payload = stats.snapshot()
+ if report.perf_metrics is None:
+ report.perf_metrics = {}
+ summary = report.perf_metrics.setdefault('summary', {})
+ summary['request'] = payload
+
def finalize(self, *args, **kwargs):
self.benchmark.finalize(*args, **kwargs)
self.cache_manager.close()
diff --git a/evalscope/evalscope/models/anthropic_compatible.py b/evalscope/evalscope/models/anthropic_compatible.py
index cf7cfca..e6a9ca0 100644
--- a/evalscope/evalscope/models/anthropic_compatible.py
+++ b/evalscope/evalscope/models/anthropic_compatible.py
@@ -172,42 +172,44 @@ class AnthropicCompatibleAPI(ModelAPI):
self.validate_request_params(request)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- # Generate completion
- message = retry_call(
- self.client.messages.create,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
-
- # Handle streaming response
- if not isinstance(message, Message):
- message, ttft = collect_stream_response(message, request_start=t_start)
-
- total_time = time.monotonic() - t_start
-
- response = message.model_dump()
- self.on_response(response)
-
- # Build output and populate timing + perf metrics
- choices = self.chat_choices_from_message(message, tools)
- output = model_output_from_anthropic(message, choices)
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except (BadRequestError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ # Generate completion
+ message = retry_call(
+ self.client.messages.create,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+
+ # Handle streaming response
+ if not isinstance(message, Message):
+ message, ttft = collect_stream_response(message, request_start=t_start)
+
+ total_time = time.monotonic() - t_start
+
+ response = message.model_dump()
+ self.on_response(response)
+
+ # Build output and populate timing + perf metrics
+ choices = self.chat_choices_from_message(message, tools)
+ output = model_output_from_anthropic(message, choices)
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except (BadRequestError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
async def generate_async(
self,
@@ -256,41 +258,43 @@ class AnthropicCompatibleAPI(ModelAPI):
self.validate_request_params(request)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- # Async generation with retry
- message = await async_retry_call(
- self.async_client.messages.create,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
-
- # Handle streaming response
- if not isinstance(message, Message):
- message, ttft = await async_collect_stream_response(message, request_start=t_start)
-
- total_time = time.monotonic() - t_start
-
- response = message.model_dump()
- self.on_response(response)
-
- choices = self.chat_choices_from_message(message, tools)
- output = model_output_from_anthropic(message, choices)
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except (BadRequestError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ # Async generation with retry
+ message = await async_retry_call(
+ self.async_client.messages.create,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+
+ # Handle streaming response
+ if not isinstance(message, Message):
+ message, ttft = await async_collect_stream_response(message, request_start=t_start)
+
+ total_time = time.monotonic() - t_start
+
+ response = message.model_dump()
+ self.on_response(response)
+
+ choices = self.chat_choices_from_message(message, tools)
+ output = model_output_from_anthropic(message, choices)
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except (BadRequestError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice,
config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
diff --git a/evalscope/evalscope/models/litellm_compatible.py b/evalscope/evalscope/models/litellm_compatible.py
index 55af148..ef9c0fc 100644
--- a/evalscope/evalscope/models/litellm_compatible.py
+++ b/evalscope/evalscope/models/litellm_compatible.py
@@ -87,40 +87,42 @@ class LiteLLMAPI(ModelAPI):
if self.base_url:
request['api_base'] = self.base_url
- try:
- t_start = time.monotonic()
-
- response = retry_call(
- litellm.completion,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
-
- total_time = time.monotonic() - t_start
- ttft: Optional[float] = None
-
- if config.stream and not isinstance(response, ChatCompletion):
- completion, ttft = collect_stream_response(response, request_start=t_start)
- else:
- completion = ChatCompletion(**response.model_dump())
-
- choices = chat_choices_from_openai(completion, tools)
- output = model_output_from_openai(completion, choices)
-
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except Exception as ex:
- logger.error(f'LiteLLM [{self.model_name}] error: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+
+ response = retry_call(
+ litellm.completion,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+
+ total_time = time.monotonic() - t_start
+ ttft: Optional[float] = None
+
+ if config.stream and not isinstance(response, ChatCompletion):
+ completion, ttft = collect_stream_response(response, request_start=t_start)
+ else:
+ completion = ChatCompletion(**response.model_dump())
+
+ choices = chat_choices_from_openai(completion, tools)
+ output = model_output_from_openai(completion, choices)
+
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except Exception as ex:
+ logger.error(f'LiteLLM [{self.model_name}] error: {ex}')
+ raise
async def generate_async(
self,
@@ -157,38 +159,40 @@ class LiteLLMAPI(ModelAPI):
if self.base_url:
request['api_base'] = self.base_url
- try:
- t_start = time.monotonic()
-
- # Async generation with retry
- response = await async_retry_call(
- litellm.acompletion,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
-
- total_time = time.monotonic() - t_start
- ttft: Optional[float] = None
-
- if config.stream and not isinstance(response, ChatCompletion):
- completion, ttft = await async_collect_stream_response(response, request_start=t_start)
- else:
- completion = ChatCompletion(**response.model_dump())
-
- choices = chat_choices_from_openai(completion, tools)
- output = model_output_from_openai(completion, choices)
-
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except Exception as ex:
- logger.error(f'LiteLLM [{self.model_name}] async error: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+
+ # Async generation with retry
+ response = await async_retry_call(
+ litellm.acompletion,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+
+ total_time = time.monotonic() - t_start
+ ttft: Optional[float] = None
+
+ if config.stream and not isinstance(response, ChatCompletion):
+ completion, ttft = await async_collect_stream_response(response, request_start=t_start)
+ else:
+ completion = ChatCompletion(**response.model_dump())
+
+ choices = chat_choices_from_openai(completion, tools)
+ output = model_output_from_openai(completion, choices)
+
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except Exception as ex:
+ logger.error(f'LiteLLM [{self.model_name}] async error: {ex}')
+ raise
diff --git a/evalscope/evalscope/models/openai_compatible.py b/evalscope/evalscope/models/openai_compatible.py
index 2e84cbb..1dcdd43 100644
--- a/evalscope/evalscope/models/openai_compatible.py
+++ b/evalscope/evalscope/models/openai_compatible.py
@@ -140,51 +140,53 @@ class OpenAICompatibleAPI(ModelAPI):
self.validate_request_params(request)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- # A streaming request is not complete when create() returns: the
- # connection may still fail while its chunks are being consumed.
- # Retry the whole request so a partial response is discarded and
- # replaced by one complete response.
- def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
- raw_completion = self.client.chat.completions.create(**request)
- if isinstance(raw_completion, ChatCompletion):
- return raw_completion, None
- return collect_stream_response(raw_completion, request_start=t_start)
-
- completion, ttft = retry_call(
- _create_and_collect,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- )
-
- total_time = time.monotonic() - t_start
-
- response = completion.model_dump()
- self.on_response(response)
-
- # return output and call
- choices = self.chat_choices_from_completion(completion, tools)
- output = model_output_from_openai(completion, choices)
-
- # Populate timing fields
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
- except ValueError as ex:
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ # A streaming request is not complete when create() returns: the
+ # connection may still fail while its chunks are being consumed.
+ # Retry the whole request so a partial response is discarded and
+ # replaced by one complete response.
+ def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
+ raw_completion = self.client.chat.completions.create(**request)
+ if isinstance(raw_completion, ChatCompletion):
+ return raw_completion, None
+ return collect_stream_response(raw_completion, request_start=t_start)
+
+ completion, ttft = retry_call(
+ _create_and_collect,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ )
+
+ total_time = time.monotonic() - t_start
+
+ response = completion.model_dump()
+ self.on_response(response)
+
+ # return output and call
+ choices = self.chat_choices_from_completion(completion, tools)
+ output = model_output_from_openai(completion, choices)
+
+ # Populate timing fields
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
+ except ValueError as ex:
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
+ raise
async def generate_async(
self,
@@ -219,49 +221,51 @@ class OpenAICompatibleAPI(ModelAPI):
self.validate_request_params(request)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- # Keep stream consumption inside the retry boundary. If an async
- # stream is interrupted, start a fresh request rather than
- # returning or persisting its partial response.
- async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
- raw_completion = await self.async_client.chat.completions.create(**request)
- if isinstance(raw_completion, ChatCompletion):
- return raw_completion, None
- return await async_collect_stream_response(raw_completion, request_start=t_start)
-
- completion, ttft = await async_retry_call(
- _create_and_collect,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- )
-
- total_time = time.monotonic() - t_start
-
- response = completion.model_dump()
- self.on_response(response)
-
- # Return output
- choices = self.chat_choices_from_completion(completion, tools)
- output = model_output_from_openai(completion, choices)
-
- output.time = total_time
- usage = output.usage
- output.message.perf_metrics = PerformanceMetrics(
- latency=total_time,
- ttft=ttft,
- input_tokens=usage.input_tokens if usage else 0,
- output_tokens=usage.output_tokens if usage else 0,
- )
- return output
-
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
- except ValueError as ex:
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ # Keep stream consumption inside the retry boundary. If an async
+ # stream is interrupted, start a fresh request rather than
+ # returning or persisting its partial response.
+ async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
+ raw_completion = await self.async_client.chat.completions.create(**request)
+ if isinstance(raw_completion, ChatCompletion):
+ return raw_completion, None
+ return await async_collect_stream_response(raw_completion, request_start=t_start)
+
+ completion, ttft = await async_retry_call(
+ _create_and_collect,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ )
+
+ total_time = time.monotonic() - t_start
+
+ response = completion.model_dump()
+ self.on_response(response)
+
+ # Return output
+ choices = self.chat_choices_from_completion(completion, tools)
+ output = model_output_from_openai(completion, choices)
+
+ output.time = total_time
+ usage = output.usage
+ output.message.perf_metrics = PerformanceMetrics(
+ latency=total_time,
+ ttft=ttft,
+ input_tokens=usage.input_tokens if usage else 0,
+ output_tokens=usage.output_tokens if usage else 0,
+ )
+ return output
+
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
+ except ValueError as ex:
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
+ raise
def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice,
config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
diff --git a/evalscope/evalscope/models/openai_responses.py b/evalscope/evalscope/models/openai_responses.py
index f28c008..b8e0652 100644
--- a/evalscope/evalscope/models/openai_responses.py
+++ b/evalscope/evalscope/models/openai_responses.py
@@ -74,27 +74,29 @@ class OpenAIResponsesAPI(OpenAICompatibleAPI):
) -> ModelOutput:
request, tools, config = self._build_request(input, tools, tool_choice, config)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- response = retry_call(
- self.client.responses.create,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
- if not self._is_response_object(response):
- response, ttft = collect_response_stream(response, request_start=t_start)
-
- total_time = time.monotonic() - t_start
- return self._build_output(response, tools, total_time, ttft)
-
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
- except ValueError as ex:
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ response = retry_call(
+ self.client.responses.create,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+ if not self._is_response_object(response):
+ response, ttft = collect_response_stream(response, request_start=t_start)
+
+ total_time = time.monotonic() - t_start
+ return self._build_output(response, tools, total_time, ttft)
+
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
+ except ValueError as ex:
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
+ raise
async def generate_async(
self,
@@ -105,27 +107,29 @@ class OpenAIResponsesAPI(OpenAICompatibleAPI):
) -> ModelOutput:
request, tools, config = self._build_request(input, tools, tool_choice, config)
- try:
- t_start = time.monotonic()
- ttft: Optional[float] = None
-
- response = await async_retry_call(
- self.async_client.responses.create,
- retries=config.retries,
- sleep_interval=config.retry_interval,
- **request,
- )
- if not self._is_response_object(response):
- response, ttft = await async_collect_response_stream(response, request_start=t_start)
-
- total_time = time.monotonic() - t_start
- return self._build_output(response, tools, total_time, ttft)
-
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
- return self.handle_bad_request(ex)
- except ValueError as ex:
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
- raise
+ with self._track_logical_request():
+ try:
+ t_start = time.monotonic()
+ ttft: Optional[float] = None
+
+ response = await async_retry_call(
+ self.async_client.responses.create,
+ retries=config.retries,
+ sleep_interval=config.retry_interval,
+ on_attempt=self.request_stats.on_attempt,
+ **request,
+ )
+ if not self._is_response_object(response):
+ response, ttft = await async_collect_response_stream(response, request_start=t_start)
+
+ total_time = time.monotonic() - t_start
+ return self._build_output(response, tools, total_time, ttft)
+
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+ return self.handle_bad_request(ex)
+ except ValueError as ex:
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
+ raise
def _build_request(
self,
diff --git a/evalscope/evalscope/report/combinator.py b/evalscope/evalscope/report/combinator.py
index dee00b8..0b44a31 100644
--- a/evalscope/evalscope/report/combinator.py
+++ b/evalscope/evalscope/report/combinator.py
@@ -18,6 +18,15 @@ Combine and generate table for reports of LLMs.
"""
+def _format_request_success(summary: Dict[str, Any]) -> str:
+ """Format vendor HTTP success rate from ``perf_metrics.summary.request``."""
+ request = summary.get('request') or {}
+ rate = request.get('success_rate')
+ if rate is None:
+ return '-'
+ return f'{float(rate) * 100:.1f}%'
+
+
def _is_report_json(data: Any) -> bool:
if not isinstance(data, dict):
return False
@@ -254,6 +263,7 @@ def gen_perf_table(
'Model': report.model_name,
'Dataset': report.dataset_name,
'Num': ps.n_samples,
+ 'Req Succ%': _format_request_success(summary),
'Avg Lat\n(s)': round(ps.avg_latency, 4),
'Avg TTFT\n(ms)': round(ps.avg_ttft * 1000, 2) if ps.avg_ttft is not None else '-',
'Avg TPOT\n(ms)': round(ps.avg_tpot * 1000, 2) if ps.avg_tpot is not None else '-',
diff --git a/evalscope/evalscope/utils/function_utils.py b/evalscope/evalscope/utils/function_utils.py
index 2ae29e2..05037d2 100644
--- a/evalscope/evalscope/utils/function_utils.py
+++ b/evalscope/evalscope/utils/function_utils.py
@@ -68,12 +68,22 @@ def run_once(func: Callable[..., T]) -> Callable[..., T]:
return wrapper
-def retry_call(func, *args, retries=3, sleep_interval=0, **kwargs):
- """Function that retries a function call up to `retries` times if an exception occurs."""
+def retry_call(func, *args, retries=3, sleep_interval=0, on_attempt=None, **kwargs):
+ """Function that retries a function call up to `retries` times if an exception occurs.
+
+ ``on_attempt(ok, exc=None)`` is invoked once per try (including retries) so
+ callers can count vendor HTTP success/failure independently of the final
+ ``generate()`` outcome.
+ """
for attempt in range(retries):
try:
- return func(*args, **kwargs)
+ result = func(*args, **kwargs)
+ if on_attempt is not None:
+ on_attempt(True, None)
+ return result
except Exception as e:
+ if on_attempt is not None:
+ on_attempt(False, e)
if attempt < retries - 1:
if sleep_interval > 0:
logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...')
@@ -83,13 +93,23 @@ def retry_call(func, *args, retries=3, sleep_interval=0, **kwargs):
async def async_retry_call(
- func: Callable[..., Awaitable[T]], *args, retries: int = 3, sleep_interval: float = 0, **kwargs
+ func: Callable[..., Awaitable[T]],
+ *args,
+ retries: int = 3,
+ sleep_interval: float = 0,
+ on_attempt=None,
+ **kwargs,
) -> T:
"""Async version of retry_call. Retries an async function call up to `retries` times if an exception occurs."""
for attempt in range(retries):
try:
- return await func(*args, **kwargs)
+ result = await func(*args, **kwargs)
+ if on_attempt is not None:
+ on_attempt(True, None)
+ return result
except Exception as e:
+ if on_attempt is not None:
+ on_attempt(False, e)
if attempt < retries - 1:
if sleep_interval > 0:
logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...')
Reference in New Issue View Git Blame Copy Permalink