diff --git a/evalscope/evalscope/api/messages/__init__.py b/evalscope/evalscope/api/messages/__init__.py index c79fac4..b606da7 100644 --- a/evalscope/evalscope/api/messages/__init__.py +++ b/evalscope/evalscope/api/messages/__init__.py @@ -11,4 +11,5 @@ from .chat_message import ( ) from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo from .perf_metrics import PerformanceMetrics, PerfSummary +from .request_stats import RequestStats, is_client_error from .utils import parse_content_with_reasoning diff --git a/evalscope/evalscope/api/model/model.py b/evalscope/evalscope/api/model/model.py index b0231e8..b14cd0f 100644 --- a/evalscope/evalscope/api/model/model.py +++ b/evalscope/evalscope/api/model/model.py @@ -1,10 +1,12 @@ import abc import asyncio +from contextlib import contextmanager from functools import partial from pydantic_core import to_jsonable_python from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional, Sequence, Union from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser +from evalscope.api.messages.request_stats import RequestStats from evalscope.api.registry import get_model_api from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo from evalscope.utils import get_logger, get_secret_value @@ -43,6 +45,18 @@ class ModelAPI(abc.ABC): self.base_url = base_url self.api_key = api_key self.config = config + self.request_stats = RequestStats() + + @contextmanager + def _track_logical_request(self): + """Count one logical generate() call after HTTP retries have finished.""" + try: + yield + except Exception: + self.request_stats.record_logical(ok=False) + raise + else: + self.request_stats.record_logical(ok=True) @abc.abstractmethod def generate( diff --git a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py index 63b5f73..bc11e03 100644 --- a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py +++ b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py @@ -51,6 +51,16 @@ COMMON_EXTRA_PARAMS = { 'description': 'Extra kwargs passed to Pier AgentConfig.kwargs.', 'value': {}, }, + 'pier_model_prefix': { + 'type': 'str', + 'description': 'LiteLLM provider prefix prepended when TaskConfig.model has no slash (e.g. openai).', + 'value': 'openai', + }, + 'pier_model_name': { + 'type': 'str', + 'description': 'Optional full Pier model name provider/model. Overrides TaskConfig.model when set.', + 'value': '', + }, } @@ -65,6 +75,8 @@ class DeepSWEAdapter(AgentAdapter): self.categories = self._as_list(extra_params.get('categories') or []) self.sample_seed = extra_params.get('sample_seed') self.pier_agent_kwargs = dict(extra_params.get('pier_agent_kwargs') or {}) + self.pier_model_prefix = str(extra_params.get('pier_model_prefix') or 'openai') + self.pier_model_name = str(extra_params.get('pier_model_name') or '') @staticmethod def _as_list(value: Union[str, List[Any], Tuple[Any, ...]]) -> List[str]: @@ -127,6 +139,8 @@ class DeepSWEAdapter(AgentAdapter): from pier.models.trial.config import AgentConfig, EnvironmentConfig, TaskConfig, VerifierConfig task_id = str(sample.metadata['task_id']) + pier_kwargs = dict(self.pier_agent_kwargs) + pier_kwargs.setdefault('model_class', 'litellm') config = JobConfig( job_name=f'{task_id[:48].rstrip("_-")}__{uuid.uuid4().hex[:8]}', @@ -140,9 +154,9 @@ class DeepSWEAdapter(AgentAdapter): environment_build_timeout_multiplier=1.0, agents=[AgentConfig( name='mini-swe-agent', - model_name=model.name, - kwargs=self.pier_agent_kwargs, - env={}, + model_name=self._pier_model_name(model), + kwargs=pier_kwargs, + env=self._pier_agent_env(), )], environment=EnvironmentConfig(type='docker'), verifier=VerifierConfig(env={}), @@ -159,6 +173,38 @@ class DeepSWEAdapter(AgentAdapter): self._raise_for_pier_failures(result_dict) return result_dict + def _pier_model_name(self, model: Model) -> str: + name = self.pier_model_name or (model.name if model else '') + name = str(name).strip() + if name and '/' not in name: + name = f'{self.pier_model_prefix}/{name}' + return name + + @staticmethod + def _plain_secret(value: Any) -> str: + if value is None: + return '' + if hasattr(value, 'get_secret_value'): + value = value.get_secret_value() + return str(value).strip() + + def _pier_agent_env(self) -> Dict[str, str]: + env: Dict[str, str] = { + 'LITELLM_LOCAL_MODEL_COST_MAP': 'true', + } + tc = self._task_config + if tc is None: + return env + api_url = self._plain_secret(getattr(tc, 'api_url', None)) + api_key = self._plain_secret(getattr(tc, 'api_key', None)) + if api_url: + env['OPENAI_API_BASE'] = api_url + env['OPENAI_BASE_URL'] = api_url + if api_key and api_key != 'EMPTY': + env['OPENAI_API_KEY'] = api_key + env['MSWEA_API_KEY'] = api_key + return env + @staticmethod def _raise_for_pier_failures(result_dict: Dict[str, Any]) -> None: trial_results = result_dict.get('trial_results') or [] @@ -214,6 +260,8 @@ integrates it through Pier and runs each benchmark sample as one Pier Python API - Dataset defaults to ModelScope `evalscope/deep-swe` - DeepSWE runs through Pier's Docker environment in EvalScope - Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API +- If `TaskConfig.model` has no slash, EvalScope prefixes `pier_model_prefix` (default `openai/`) for Pier/LiteLLM +- API base/key are forwarded as `OPENAI_API_BASE` / `OPENAI_API_KEY` from TaskConfig """, dataset_id=DEFAULT_MODELSCOPE_DATASET_ID, eval_split='test', diff --git a/evalscope/evalscope/evaluator/evaluator.py b/evalscope/evalscope/evaluator/evaluator.py index 68a0987..2272503 100644 --- a/evalscope/evalscope/evaluator/evaluator.py +++ b/evalscope/evalscope/evaluator/evaluator.py @@ -534,12 +534,15 @@ class DefaultEvaluator(Evaluator): if self.task_config.collect_perf: report.perf_metrics = self.perf_collector.get_perf_dict() or None + # Vendor HTTP request success rate (counted per attempt, including retries). + self._inject_request_stats(report) + # Save the complete report to file report.to_json(report_file) logger.info(f'Dump report to: {report_file} \n') # Print per-benchmark perf table when perf data is available - if self.task_config.collect_perf and report.perf_metrics: + if report.perf_metrics: try: perf_table = gen_perf_table(report_list=[report]) if perf_table: @@ -549,6 +552,18 @@ class DefaultEvaluator(Evaluator): return report + def _inject_request_stats(self, report: Report) -> None: + """Attach HTTP request success stats under ``perf_metrics.summary.request``.""" + api = getattr(self.model, 'api', None) + stats = getattr(api, 'request_stats', None) + if stats is None or stats.total_attempts <= 0: + return + payload = stats.snapshot() + if report.perf_metrics is None: + report.perf_metrics = {} + summary = report.perf_metrics.setdefault('summary', {}) + summary['request'] = payload + def finalize(self, *args, **kwargs): self.benchmark.finalize(*args, **kwargs) self.cache_manager.close() diff --git a/evalscope/evalscope/models/anthropic_compatible.py b/evalscope/evalscope/models/anthropic_compatible.py index cf7cfca..e6a9ca0 100644 --- a/evalscope/evalscope/models/anthropic_compatible.py +++ b/evalscope/evalscope/models/anthropic_compatible.py @@ -172,42 +172,44 @@ class AnthropicCompatibleAPI(ModelAPI): self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Generate completion - message = retry_call( - self.client.messages.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - # Handle streaming response - if not isinstance(message, Message): - message, ttft = collect_stream_response(message, request_start=t_start) - - total_time = time.monotonic() - t_start - - response = message.model_dump() - self.on_response(response) - - # Build output and populate timing + perf metrics - choices = self.chat_choices_from_message(message, tools) - output = model_output_from_anthropic(message, choices) - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Generate completion + message = retry_call( + self.client.messages.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + # Handle streaming response + if not isinstance(message, Message): + message, ttft = collect_stream_response(message, request_start=t_start) + + total_time = time.monotonic() - t_start + + response = message.model_dump() + self.on_response(response) + + # Build output and populate timing + perf metrics + choices = self.chat_choices_from_message(message, tools) + output = model_output_from_anthropic(message, choices) + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) async def generate_async( self, @@ -256,41 +258,43 @@ class AnthropicCompatibleAPI(ModelAPI): self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Async generation with retry - message = await async_retry_call( - self.async_client.messages.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - # Handle streaming response - if not isinstance(message, Message): - message, ttft = await async_collect_stream_response(message, request_start=t_start) - - total_time = time.monotonic() - t_start - - response = message.model_dump() - self.on_response(response) - - choices = self.chat_choices_from_message(message, tools) - output = model_output_from_anthropic(message, choices) - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Async generation with retry + message = await async_retry_call( + self.async_client.messages.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + # Handle streaming response + if not isinstance(message, Message): + message, ttft = await async_collect_stream_response(message, request_start=t_start) + + total_time = time.monotonic() - t_start + + response = message.model_dump() + self.on_response(response) + + choices = self.chat_choices_from_message(message, tools) + output = model_output_from_anthropic(message, choices) + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice, config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]: diff --git a/evalscope/evalscope/models/litellm_compatible.py b/evalscope/evalscope/models/litellm_compatible.py index 55af148..ef9c0fc 100644 --- a/evalscope/evalscope/models/litellm_compatible.py +++ b/evalscope/evalscope/models/litellm_compatible.py @@ -87,40 +87,42 @@ class LiteLLMAPI(ModelAPI): if self.base_url: request['api_base'] = self.base_url - try: - t_start = time.monotonic() - - response = retry_call( - litellm.completion, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - total_time = time.monotonic() - t_start - ttft: Optional[float] = None - - if config.stream and not isinstance(response, ChatCompletion): - completion, ttft = collect_stream_response(response, request_start=t_start) - else: - completion = ChatCompletion(**response.model_dump()) - - choices = chat_choices_from_openai(completion, tools) - output = model_output_from_openai(completion, choices) - - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except Exception as ex: - logger.error(f'LiteLLM [{self.model_name}] error: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + + response = retry_call( + litellm.completion, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + total_time = time.monotonic() - t_start + ttft: Optional[float] = None + + if config.stream and not isinstance(response, ChatCompletion): + completion, ttft = collect_stream_response(response, request_start=t_start) + else: + completion = ChatCompletion(**response.model_dump()) + + choices = chat_choices_from_openai(completion, tools) + output = model_output_from_openai(completion, choices) + + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except Exception as ex: + logger.error(f'LiteLLM [{self.model_name}] error: {ex}') + raise async def generate_async( self, @@ -157,38 +159,40 @@ class LiteLLMAPI(ModelAPI): if self.base_url: request['api_base'] = self.base_url - try: - t_start = time.monotonic() - - # Async generation with retry - response = await async_retry_call( - litellm.acompletion, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - total_time = time.monotonic() - t_start - ttft: Optional[float] = None - - if config.stream and not isinstance(response, ChatCompletion): - completion, ttft = await async_collect_stream_response(response, request_start=t_start) - else: - completion = ChatCompletion(**response.model_dump()) - - choices = chat_choices_from_openai(completion, tools) - output = model_output_from_openai(completion, choices) - - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except Exception as ex: - logger.error(f'LiteLLM [{self.model_name}] async error: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + + # Async generation with retry + response = await async_retry_call( + litellm.acompletion, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + total_time = time.monotonic() - t_start + ttft: Optional[float] = None + + if config.stream and not isinstance(response, ChatCompletion): + completion, ttft = await async_collect_stream_response(response, request_start=t_start) + else: + completion = ChatCompletion(**response.model_dump()) + + choices = chat_choices_from_openai(completion, tools) + output = model_output_from_openai(completion, choices) + + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except Exception as ex: + logger.error(f'LiteLLM [{self.model_name}] async error: {ex}') + raise diff --git a/evalscope/evalscope/models/openai_compatible.py b/evalscope/evalscope/models/openai_compatible.py index 2e84cbb..1dcdd43 100644 --- a/evalscope/evalscope/models/openai_compatible.py +++ b/evalscope/evalscope/models/openai_compatible.py @@ -140,51 +140,53 @@ class OpenAICompatibleAPI(ModelAPI): self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # A streaming request is not complete when create() returns: the - # connection may still fail while its chunks are being consumed. - # Retry the whole request so a partial response is discarded and - # replaced by one complete response. - def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: - raw_completion = self.client.chat.completions.create(**request) - if isinstance(raw_completion, ChatCompletion): - return raw_completion, None - return collect_stream_response(raw_completion, request_start=t_start) - - completion, ttft = retry_call( - _create_and_collect, - retries=config.retries, - sleep_interval=config.retry_interval, - ) - - total_time = time.monotonic() - t_start - - response = completion.model_dump() - self.on_response(response) - - # return output and call - choices = self.chat_choices_from_completion(completion, tools) - output = model_output_from_openai(completion, choices) - - # Populate timing fields - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # A streaming request is not complete when create() returns: the + # connection may still fail while its chunks are being consumed. + # Retry the whole request so a partial response is discarded and + # replaced by one complete response. + def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: + raw_completion = self.client.chat.completions.create(**request) + if isinstance(raw_completion, ChatCompletion): + return raw_completion, None + return collect_stream_response(raw_completion, request_start=t_start) + + completion, ttft = retry_call( + _create_and_collect, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + ) + + total_time = time.monotonic() - t_start + + response = completion.model_dump() + self.on_response(response) + + # return output and call + choices = self.chat_choices_from_completion(completion, tools) + output = model_output_from_openai(completion, choices) + + # Populate timing fields + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise async def generate_async( self, @@ -219,49 +221,51 @@ class OpenAICompatibleAPI(ModelAPI): self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Keep stream consumption inside the retry boundary. If an async - # stream is interrupted, start a fresh request rather than - # returning or persisting its partial response. - async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: - raw_completion = await self.async_client.chat.completions.create(**request) - if isinstance(raw_completion, ChatCompletion): - return raw_completion, None - return await async_collect_stream_response(raw_completion, request_start=t_start) - - completion, ttft = await async_retry_call( - _create_and_collect, - retries=config.retries, - sleep_interval=config.retry_interval, - ) - - total_time = time.monotonic() - t_start - - response = completion.model_dump() - self.on_response(response) - - # Return output - choices = self.chat_choices_from_completion(completion, tools) - output = model_output_from_openai(completion, choices) - - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Keep stream consumption inside the retry boundary. If an async + # stream is interrupted, start a fresh request rather than + # returning or persisting its partial response. + async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: + raw_completion = await self.async_client.chat.completions.create(**request) + if isinstance(raw_completion, ChatCompletion): + return raw_completion, None + return await async_collect_stream_response(raw_completion, request_start=t_start) + + completion, ttft = await async_retry_call( + _create_and_collect, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + ) + + total_time = time.monotonic() - t_start + + response = completion.model_dump() + self.on_response(response) + + # Return output + choices = self.chat_choices_from_completion(completion, tools) + output = model_output_from_openai(completion, choices) + + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice, config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]: diff --git a/evalscope/evalscope/models/openai_responses.py b/evalscope/evalscope/models/openai_responses.py index f28c008..b8e0652 100644 --- a/evalscope/evalscope/models/openai_responses.py +++ b/evalscope/evalscope/models/openai_responses.py @@ -74,27 +74,29 @@ class OpenAIResponsesAPI(OpenAICompatibleAPI): ) -> ModelOutput: request, tools, config = self._build_request(input, tools, tool_choice, config) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - response = retry_call( - self.client.responses.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - if not self._is_response_object(response): - response, ttft = collect_response_stream(response, request_start=t_start) - - total_time = time.monotonic() - t_start - return self._build_output(response, tools, total_time, ttft) - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + response = retry_call( + self.client.responses.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + if not self._is_response_object(response): + response, ttft = collect_response_stream(response, request_start=t_start) + + total_time = time.monotonic() - t_start + return self._build_output(response, tools, total_time, ttft) + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise async def generate_async( self, @@ -105,27 +107,29 @@ class OpenAIResponsesAPI(OpenAICompatibleAPI): ) -> ModelOutput: request, tools, config = self._build_request(input, tools, tool_choice, config) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - response = await async_retry_call( - self.async_client.responses.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - if not self._is_response_object(response): - response, ttft = await async_collect_response_stream(response, request_start=t_start) - - total_time = time.monotonic() - t_start - return self._build_output(response, tools, total_time, ttft) - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + response = await async_retry_call( + self.async_client.responses.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + if not self._is_response_object(response): + response, ttft = await async_collect_response_stream(response, request_start=t_start) + + total_time = time.monotonic() - t_start + return self._build_output(response, tools, total_time, ttft) + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise def _build_request( self, diff --git a/evalscope/evalscope/report/combinator.py b/evalscope/evalscope/report/combinator.py index dee00b8..0b44a31 100644 --- a/evalscope/evalscope/report/combinator.py +++ b/evalscope/evalscope/report/combinator.py @@ -18,6 +18,15 @@ Combine and generate table for reports of LLMs. """ +def _format_request_success(summary: Dict[str, Any]) -> str: + """Format vendor HTTP success rate from ``perf_metrics.summary.request``.""" + request = summary.get('request') or {} + rate = request.get('success_rate') + if rate is None: + return '-' + return f'{float(rate) * 100:.1f}%' + + def _is_report_json(data: Any) -> bool: if not isinstance(data, dict): return False @@ -254,6 +263,7 @@ def gen_perf_table( 'Model': report.model_name, 'Dataset': report.dataset_name, 'Num': ps.n_samples, + 'Req Succ%': _format_request_success(summary), 'Avg Lat\n(s)': round(ps.avg_latency, 4), 'Avg TTFT\n(ms)': round(ps.avg_ttft * 1000, 2) if ps.avg_ttft is not None else '-', 'Avg TPOT\n(ms)': round(ps.avg_tpot * 1000, 2) if ps.avg_tpot is not None else '-', diff --git a/evalscope/evalscope/utils/function_utils.py b/evalscope/evalscope/utils/function_utils.py index 2ae29e2..05037d2 100644 --- a/evalscope/evalscope/utils/function_utils.py +++ b/evalscope/evalscope/utils/function_utils.py @@ -68,12 +68,22 @@ def run_once(func: Callable[..., T]) -> Callable[..., T]: return wrapper -def retry_call(func, *args, retries=3, sleep_interval=0, **kwargs): - """Function that retries a function call up to `retries` times if an exception occurs.""" +def retry_call(func, *args, retries=3, sleep_interval=0, on_attempt=None, **kwargs): + """Function that retries a function call up to `retries` times if an exception occurs. + + ``on_attempt(ok, exc=None)`` is invoked once per try (including retries) so + callers can count vendor HTTP success/failure independently of the final + ``generate()`` outcome. + """ for attempt in range(retries): try: - return func(*args, **kwargs) + result = func(*args, **kwargs) + if on_attempt is not None: + on_attempt(True, None) + return result except Exception as e: + if on_attempt is not None: + on_attempt(False, e) if attempt < retries - 1: if sleep_interval > 0: logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...') @@ -83,13 +93,23 @@ def retry_call(func, *args, retries=3, sleep_interval=0, **kwargs): async def async_retry_call( - func: Callable[..., Awaitable[T]], *args, retries: int = 3, sleep_interval: float = 0, **kwargs + func: Callable[..., Awaitable[T]], + *args, + retries: int = 3, + sleep_interval: float = 0, + on_attempt=None, + **kwargs, ) -> T: """Async version of retry_call. Retries an async function call up to `retries` times if an exception occurs.""" for attempt in range(retries): try: - return await func(*args, **kwargs) + result = await func(*args, **kwargs) + if on_attempt is not None: + on_attempt(True, None) + return result except Exception as e: + if on_attempt is not None: + on_attempt(False, e) if attempt < retries - 1: if sleep_interval > 0: logger.warning(f'Attempt {attempt + 1} / {retries} failed: {e}. Retrying...')