13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
156 lines
6.0 KiB
Diff
156 lines
6.0 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/models/litellm_compatible.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/models/litellm_compatible.py 2026-09-01 03:48:30.955904239 +0000
|
|
@@ -87,40 +87,42 @@
|
|
if self.base_url:
|
|
request['api_base'] = self.base_url
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
-
|
|
- response = retry_call(
|
|
- litellm.completion,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- **request,
|
|
- )
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- if config.stream and not isinstance(response, ChatCompletion):
|
|
- completion, ttft = collect_stream_response(response, request_start=t_start)
|
|
- else:
|
|
- completion = ChatCompletion(**response.model_dump())
|
|
-
|
|
- choices = chat_choices_from_openai(completion, tools)
|
|
- output = model_output_from_openai(completion, choices)
|
|
-
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
-
|
|
- except Exception as ex:
|
|
- logger.error(f'LiteLLM [{self.model_name}] error: {ex}')
|
|
- raise
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+
|
|
+ response = retry_call(
|
|
+ litellm.completion,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ **request,
|
|
+ )
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ if config.stream and not isinstance(response, ChatCompletion):
|
|
+ completion, ttft = collect_stream_response(response, request_start=t_start)
|
|
+ else:
|
|
+ completion = ChatCompletion(**response.model_dump())
|
|
+
|
|
+ choices = chat_choices_from_openai(completion, tools)
|
|
+ output = model_output_from_openai(completion, choices)
|
|
+
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
+
|
|
+ except Exception as ex:
|
|
+ logger.error(f'LiteLLM [{self.model_name}] error: {ex}')
|
|
+ raise
|
|
|
|
async def generate_async(
|
|
self,
|
|
@@ -157,38 +159,40 @@
|
|
if self.base_url:
|
|
request['api_base'] = self.base_url
|
|
|
|
- try:
|
|
- t_start = time.monotonic()
|
|
-
|
|
- # Async generation with retry
|
|
- response = await async_retry_call(
|
|
- litellm.acompletion,
|
|
- retries=config.retries,
|
|
- sleep_interval=config.retry_interval,
|
|
- **request,
|
|
- )
|
|
-
|
|
- total_time = time.monotonic() - t_start
|
|
- ttft: Optional[float] = None
|
|
-
|
|
- if config.stream and not isinstance(response, ChatCompletion):
|
|
- completion, ttft = await async_collect_stream_response(response, request_start=t_start)
|
|
- else:
|
|
- completion = ChatCompletion(**response.model_dump())
|
|
-
|
|
- choices = chat_choices_from_openai(completion, tools)
|
|
- output = model_output_from_openai(completion, choices)
|
|
-
|
|
- output.time = total_time
|
|
- usage = output.usage
|
|
- output.message.perf_metrics = PerformanceMetrics(
|
|
- latency=total_time,
|
|
- ttft=ttft,
|
|
- input_tokens=usage.input_tokens if usage else 0,
|
|
- output_tokens=usage.output_tokens if usage else 0,
|
|
- )
|
|
- return output
|
|
-
|
|
- except Exception as ex:
|
|
- logger.error(f'LiteLLM [{self.model_name}] async error: {ex}')
|
|
- raise
|
|
+ with self._track_logical_request():
|
|
+ try:
|
|
+ t_start = time.monotonic()
|
|
+
|
|
+ # Async generation with retry
|
|
+ response = await async_retry_call(
|
|
+ litellm.acompletion,
|
|
+ retries=config.retries,
|
|
+ sleep_interval=config.retry_interval,
|
|
+ on_attempt=self.request_stats.on_attempt,
|
|
+ **request,
|
|
+ )
|
|
+
|
|
+ total_time = time.monotonic() - t_start
|
|
+ ttft: Optional[float] = None
|
|
+
|
|
+ if config.stream and not isinstance(response, ChatCompletion):
|
|
+ completion, ttft = await async_collect_stream_response(response, request_start=t_start)
|
|
+ else:
|
|
+ completion = ChatCompletion(**response.model_dump())
|
|
+
|
|
+ choices = chat_choices_from_openai(completion, tools)
|
|
+ output = model_output_from_openai(completion, choices)
|
|
+
|
|
+ output.time = total_time
|
|
+ usage = output.usage
|
|
+ output.message.perf_metrics = PerformanceMetrics(
|
|
+ latency=total_time,
|
|
+ ttft=ttft,
|
|
+ input_tokens=usage.input_tokens if usage else 0,
|
|
+ output_tokens=usage.output_tokens if usage else 0,
|
|
+ )
|
|
+ return output
|
|
+
|
|
+ except Exception as ex:
|
|
+ logger.error(f'LiteLLM [{self.model_name}] async error: {ex}')
|
|
+ raise
|