13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
46 lines
1.7 KiB
Diff
46 lines
1.7 KiB
Diff
--- /tmp/evalscope-sync/src-v191/evalscope/api/model/model.py 2026-07-21 03:18:18.000000000 +0000
|
|
+++ /data1/syy/evalscope/evalstone/evalscope/evalscope/api/model/model.py 2026-09-01 03:48:30.955904239 +0000
|
|
@@ -1,10 +1,12 @@
|
|
import abc
|
|
import asyncio
|
|
+from contextlib import contextmanager
|
|
from functools import partial
|
|
from pydantic_core import to_jsonable_python
|
|
from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional, Sequence, Union
|
|
|
|
from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser
|
|
+from evalscope.api.messages.request_stats import RequestStats
|
|
from evalscope.api.registry import get_model_api
|
|
from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo
|
|
from evalscope.utils import get_logger, get_secret_value
|
|
@@ -43,6 +45,18 @@
|
|
self.base_url = base_url
|
|
self.api_key = api_key
|
|
self.config = config
|
|
+ self.request_stats = RequestStats()
|
|
+
|
|
+ @contextmanager
|
|
+ def _track_logical_request(self):
|
|
+ """Count one logical generate() call after HTTP retries have finished."""
|
|
+ try:
|
|
+ yield
|
|
+ except Exception:
|
|
+ self.request_stats.record_logical(ok=False)
|
|
+ raise
|
|
+ else:
|
|
+ self.request_stats.record_logical(ok=True)
|
|
|
|
@abc.abstractmethod
|
|
def generate(
|
|
@@ -361,6 +375,10 @@
|
|
if cached is not None:
|
|
return cached
|
|
|
|
+ # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used.
|
|
+ if api_key == 'EMPTY':
|
|
+ api_key = None
|
|
+
|
|
logger.info(
|
|
f'Creating model {model} with eval_type={eval_type} '
|
|
f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}'
|