--- /tmp/evalscope-sync/src-v191/evalscope/models/openai_compatible.py 2026-07-21 03:18:18.000000000 +0000 +++ /data1/syy/evalscope/evalstone/evalscope/evalscope/models/openai_compatible.py 2026-09-01 03:48:30.959904179 +0000 @@ -47,9 +47,8 @@ config=config, ) - # use service prefix to lookup api_key - self.api_key = api_key or os.environ.get('EVALSCOPE_API_KEY', None) - assert self.api_key, f'API key for {model_name} not found' + # use service prefix to lookup api_key; default to 'EMPTY' for local endpoints + self.api_key = api_key or os.environ.get('EVALSCOPE_API_KEY') or 'EMPTY' # use service prefix to lookup base_url self.base_url = base_url or os.environ.get('EVALSCOPE_BASE_URL', None) @@ -141,51 +140,53 @@ self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # A streaming request is not complete when create() returns: the - # connection may still fail while its chunks are being consumed. - # Retry the whole request so a partial response is discarded and - # replaced by one complete response. - def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: - raw_completion = self.client.chat.completions.create(**request) - if isinstance(raw_completion, ChatCompletion): - return raw_completion, None - return collect_stream_response(raw_completion, request_start=t_start) - - completion, ttft = retry_call( - _create_and_collect, - retries=config.retries, - sleep_interval=config.retry_interval, - ) - - total_time = time.monotonic() - t_start - - response = completion.model_dump() - self.on_response(response) - - # return output and call - choices = self.chat_choices_from_completion(completion, tools) - output = model_output_from_openai(completion, choices) - - # Populate timing fields - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # A streaming request is not complete when create() returns: the + # connection may still fail while its chunks are being consumed. + # Retry the whole request so a partial response is discarded and + # replaced by one complete response. + def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: + raw_completion = self.client.chat.completions.create(**request) + if isinstance(raw_completion, ChatCompletion): + return raw_completion, None + return collect_stream_response(raw_completion, request_start=t_start) + + completion, ttft = retry_call( + _create_and_collect, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + ) + + total_time = time.monotonic() - t_start + + response = completion.model_dump() + self.on_response(response) + + # return output and call + choices = self.chat_choices_from_completion(completion, tools) + output = model_output_from_openai(completion, choices) + + # Populate timing fields + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise async def generate_async( self, @@ -220,49 +221,51 @@ self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Keep stream consumption inside the retry boundary. If an async - # stream is interrupted, start a fresh request rather than - # returning or persisting its partial response. - async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: - raw_completion = await self.async_client.chat.completions.create(**request) - if isinstance(raw_completion, ChatCompletion): - return raw_completion, None - return await async_collect_stream_response(raw_completion, request_start=t_start) - - completion, ttft = await async_retry_call( - _create_and_collect, - retries=config.retries, - sleep_interval=config.retry_interval, - ) - - total_time = time.monotonic() - t_start - - response = completion.model_dump() - self.on_response(response) - - # Return output - choices = self.chat_choices_from_completion(completion, tools) - output = model_output_from_openai(completion, choices) - - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output - - except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) - except ValueError as ex: - logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') - raise + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Keep stream consumption inside the retry boundary. If an async + # stream is interrupted, start a fresh request rather than + # returning or persisting its partial response. + async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]: + raw_completion = await self.async_client.chat.completions.create(**request) + if isinstance(raw_completion, ChatCompletion): + return raw_completion, None + return await async_collect_stream_response(raw_completion, request_start=t_start) + + completion, ttft = await async_retry_call( + _create_and_collect, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + ) + + total_time = time.monotonic() - t_start + + response = completion.model_dump() + self.on_response(response) + + # Return output + choices = self.chat_choices_from_completion(completion, tools) + output = model_output_from_openai(completion, choices) + + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output + + except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) + except ValueError as ex: + logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}') + raise def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice, config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]: