--- /tmp/evalscope-sync/src-v191/evalscope/models/anthropic_compatible.py 2026-07-21 03:18:18.000000000 +0000 +++ /data1/syy/evalscope/evalstone/evalscope/evalscope/models/anthropic_compatible.py 2026-09-01 03:48:30.947904360 +0000 @@ -172,42 +172,44 @@ self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Generate completion - message = retry_call( - self.client.messages.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - # Handle streaming response - if not isinstance(message, Message): - message, ttft = collect_stream_response(message, request_start=t_start) - - total_time = time.monotonic() - t_start - - response = message.model_dump() - self.on_response(response) - - # Build output and populate timing + perf metrics - choices = self.chat_choices_from_message(message, tools) - output = model_output_from_anthropic(message, choices) - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Generate completion + message = retry_call( + self.client.messages.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + # Handle streaming response + if not isinstance(message, Message): + message, ttft = collect_stream_response(message, request_start=t_start) + + total_time = time.monotonic() - t_start + + response = message.model_dump() + self.on_response(response) + + # Build output and populate timing + perf metrics + choices = self.chat_choices_from_message(message, tools) + output = model_output_from_anthropic(message, choices) + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output - except (BadRequestError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) + except (BadRequestError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) async def generate_async( self, @@ -256,41 +258,43 @@ self.validate_request_params(request) - try: - t_start = time.monotonic() - ttft: Optional[float] = None - - # Async generation with retry - message = await async_retry_call( - self.async_client.messages.create, - retries=config.retries, - sleep_interval=config.retry_interval, - **request, - ) - - # Handle streaming response - if not isinstance(message, Message): - message, ttft = await async_collect_stream_response(message, request_start=t_start) - - total_time = time.monotonic() - t_start - - response = message.model_dump() - self.on_response(response) - - choices = self.chat_choices_from_message(message, tools) - output = model_output_from_anthropic(message, choices) - output.time = total_time - usage = output.usage - output.message.perf_metrics = PerformanceMetrics( - latency=total_time, - ttft=ttft, - input_tokens=usage.input_tokens if usage else 0, - output_tokens=usage.output_tokens if usage else 0, - ) - return output + with self._track_logical_request(): + try: + t_start = time.monotonic() + ttft: Optional[float] = None + + # Async generation with retry + message = await async_retry_call( + self.async_client.messages.create, + retries=config.retries, + sleep_interval=config.retry_interval, + on_attempt=self.request_stats.on_attempt, + **request, + ) + + # Handle streaming response + if not isinstance(message, Message): + message, ttft = await async_collect_stream_response(message, request_start=t_start) + + total_time = time.monotonic() - t_start + + response = message.model_dump() + self.on_response(response) + + choices = self.chat_choices_from_message(message, tools) + output = model_output_from_anthropic(message, choices) + output.time = total_time + usage = output.usage + output.message.perf_metrics = PerformanceMetrics( + latency=total_time, + ttft=ttft, + input_tokens=usage.input_tokens if usage else 0, + output_tokens=usage.output_tokens if usage else 0, + ) + return output - except (BadRequestError, PermissionDeniedError) as ex: - return self.handle_bad_request(ex) + except (BadRequestError, PermissionDeniedError) as ex: + return self.handle_bad_request(ex) def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice, config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]: