13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
209 lines
9.2 KiB
Diff
209 lines
9.2 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/models/openai_compatible.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/models/openai_compatible.py 2026-09-01 03:48:30.959904179 +0000
|
|
@@ -47,9 +47,8 @@
|
|
config=config,
|
|
)
|
|
|
|
- # use service prefix to lookup api_key
|
|
- self.api_key = api_key or os.environ.get('EVALSCOPE_API_KEY', None)
|
|
- assert self.api_key, f'API key for {model_name} not found'
|
|
+ # use service prefix to lookup api_key; default to 'EMPTY' for local endpoints
|
|
+ self.api_key = api_key or os.environ.get('EVALSCOPE_API_KEY') or 'EMPTY'
|
|
|
|
# use service prefix to lookup base_url
|
|
self.base_url = base_url or os.environ.get('EVALSCOPE_BASE_URL', None)
|
|
@@ -141,51 +140,53 @@
|
|
|
|
self.validate_request_params(request)
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- # A streaming request is not complete when create() returns: the
|
|
- # connection may still fail while its chunks are being consumed.
|
|
- # Retry the whole request so a partial response is discarded and
|
|
- # replaced by one complete response.
|
|
- def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
|
|
- raw_completion = self.client.chat.completions.create(**request)
|
|
- if isinstance(raw_completion, ChatCompletion):
|
|
- return raw_completion, None
|
|
- return collect_stream_response(raw_completion, request_start=t_start)
|
|
-
|
|
- completion, ttft = retry_call(
|
|
- _create_and_collect,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- )
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
-
|
|
- response = completion.model_dump()
|
|
- self.on_response(response)
|
|
-
|
|
- # return output and call
|
|
- choices = self.chat_choices_from_completion(completion, tools)
|
|
- output = model_output_from_openai(completion, choices)
|
|
-
|
|
- # Populate timing fields
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
-
|
|
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
|
|
- return self.handle_bad_request(ex)
|
|
- except ValueError as ex:
|
|
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
|
|
- raise
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ # A streaming request is not complete when create() returns: the
|
|
+ # connection may still fail while its chunks are being consumed.
|
|
+ # Retry the whole request so a partial response is discarded and
|
|
+ # replaced by one complete response.
|
|
+ def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
|
|
+ raw_completion = self.client.chat.completions.create(**request)
|
|
+ if isinstance(raw_completion, ChatCompletion):
|
|
+ return raw_completion, None
|
|
+ return collect_stream_response(raw_completion, request_start=t_start)
|
|
+
|
|
+ completion, ttft = retry_call(
|
|
+ _create_and_collect,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ )
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+
|
|
+ response = completion.model_dump()
|
|
+ self.on_response(response)
|
|
+
|
|
+ # return output and call
|
|
+ choices = self.chat_choices_from_completion(completion, tools)
|
|
+ output = model_output_from_openai(completion, choices)
|
|
+
|
|
+ # Populate timing fields
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
+
|
|
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
|
|
+ return self.handle_bad_request(ex)
|
|
+ except ValueError as ex:
|
|
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
|
|
+ raise
|
|
|
|
async def generate_async(
|
|
self,
|
|
@@ -220,49 +221,51 @@
|
|
|
|
self.validate_request_params(request)
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- # Keep stream consumption inside the retry boundary. If an async
|
|
- # stream is interrupted, start a fresh request rather than
|
|
- # returning or persisting its partial response.
|
|
- async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
|
|
- raw_completion = await self.async_client.chat.completions.create(**request)
|
|
- if isinstance(raw_completion, ChatCompletion):
|
|
- return raw_completion, None
|
|
- return await async_collect_stream_response(raw_completion, request_start=t_start)
|
|
-
|
|
- completion, ttft = await async_retry_call(
|
|
- _create_and_collect,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- )
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
-
|
|
- response = completion.model_dump()
|
|
- self.on_response(response)
|
|
-
|
|
- # Return output
|
|
- choices = self.chat_choices_from_completion(completion, tools)
|
|
- output = model_output_from_openai(completion, choices)
|
|
-
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
-
|
|
- except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
|
|
- return self.handle_bad_request(ex)
|
|
- except ValueError as ex:
|
|
- logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
|
|
- raise
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ # Keep stream consumption inside the retry boundary. If an async
|
|
+ # stream is interrupted, start a fresh request rather than
|
|
+ # returning or persisting its partial response.
|
|
+ async def _create_and_collect() -> Tuple[ChatCompletion, Optional[float]]:
|
|
+ raw_completion = await self.async_client.chat.completions.create(**request)
|
|
+ if isinstance(raw_completion, ChatCompletion):
|
|
+ return raw_completion, None
|
|
+ return await async_collect_stream_response(raw_completion, request_start=t_start)
|
|
+
|
|
+ completion, ttft = await async_retry_call(
|
|
+ _create_and_collect,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ )
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+
|
|
+ response = completion.model_dump()
|
|
+ self.on_response(response)
|
|
+
|
|
+ # Return output
|
|
+ choices = self.chat_choices_from_completion(completion, tools)
|
|
+ output = model_output_from_openai(completion, choices)
|
|
+
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
+
|
|
+ except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
|
|
+ return self.handle_bad_request(ex)
|
|
+ except ValueError as ex:
|
|
+ logger.error(f'Model [{self.model_name}] returned an invalid response: {ex}')
|
|
+ raise
|
|
|
|
def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice,
|
|
config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
|