13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
161 lines
6.3 KiB
Diff
161 lines
6.3 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/models/anthropic_compatible.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/models/anthropic_compatible.py 2026-09-01 03:48:30.947904360 +0000
|
|
@@ -172,42 +172,44 @@
|
|
|
|
self.validate_request_params(request)
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- # Generate completion
|
|
- message = retry_call(
|
|
- self.client.messages.create,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- **request,
|
|
- )
|
|
-
|
|
- # Handle streaming response
|
|
- if not isinstance(message, Message):
|
|
- message, ttft = collect_stream_response(message, request_start=t_start)
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
-
|
|
- response = message.model_dump()
|
|
- self.on_response(response)
|
|
-
|
|
- # Build output and populate timing + perf metrics
|
|
- choices = self.chat_choices_from_message(message, tools)
|
|
- output = model_output_from_anthropic(message, choices)
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ # Generate completion
|
|
+ message = retry_call(
|
|
+ self.client.messages.create,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ **request,
|
|
+ )
|
|
+
|
|
+ # Handle streaming response
|
|
+ if not isinstance(message, Message):
|
|
+ message, ttft = collect_stream_response(message, request_start=t_start)
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+
|
|
+ response = message.model_dump()
|
|
+ self.on_response(response)
|
|
+
|
|
+ # Build output and populate timing + perf metrics
|
|
+ choices = self.chat_choices_from_message(message, tools)
|
|
+ output = model_output_from_anthropic(message, choices)
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
|
|
- except (BadRequestError, PermissionDeniedError) as ex:
|
|
- return self.handle_bad_request(ex)
|
|
+ except (BadRequestError, PermissionDeniedError) as ex:
|
|
+ return self.handle_bad_request(ex)
|
|
|
|
async def generate_async(
|
|
self,
|
|
@@ -256,41 +258,43 @@
|
|
|
|
self.validate_request_params(request)
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- # Async generation with retry
|
|
- message = await async_retry_call(
|
|
- self.async_client.messages.create,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- **request,
|
|
- )
|
|
-
|
|
- # Handle streaming response
|
|
- if not isinstance(message, Message):
|
|
- message, ttft = await async_collect_stream_response(message, request_start=t_start)
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
-
|
|
- response = message.model_dump()
|
|
- self.on_response(response)
|
|
-
|
|
- choices = self.chat_choices_from_message(message, tools)
|
|
- output = model_output_from_anthropic(message, choices)
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ # Async generation with retry
|
|
+ message = await async_retry_call(
|
|
+ self.async_client.messages.create,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ **request,
|
|
+ )
|
|
+
|
|
+ # Handle streaming response
|
|
+ if not isinstance(message, Message):
|
|
+ message, ttft = await async_collect_stream_response(message, request_start=t_start)
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+
|
|
+ response = message.model_dump()
|
|
+ self.on_response(response)
|
|
+
|
|
+ choices = self.chat_choices_from_message(message, tools)
|
|
+ output = model_output_from_anthropic(message, choices)
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
|
|
- except (BadRequestError, PermissionDeniedError) as ex:
|
|
- return self.handle_bad_request(ex)
|
|
+ except (BadRequestError, PermissionDeniedError) as ex:
|
|
+ return self.handle_bad_request(ex)
|
|
|
|
def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice,
|
|
config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
|