From ad61a2c44e380452ea1f2b2aa873f3e49a04dde2 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Tue, 8 Sep 2026 07:32:07 +0000 Subject: [PATCH] Fix DeepSWE Pier routing and Terminal-Bench context/apt setup. Keep generation max_tokens from being used as terminus-2 context, rewrite Debian/Ubuntu .sources to the Tsinghua mirror, and skip injecting truncation_tokens into DeepSWE extra_params. Co-authored-by: Cursor --- bash/run.py | 8 + config/dpv4-int8_nothinking.yaml | 14 +- config/dpv4-int8_thinking.yaml | 6 +- .../benchmarks/deep_swe/deep_swe_adapter.py | 162 ++++++++++++++++-- .../terminal_bench/terminal_bench_adapter.py | 85 +++++++-- .../benchmarks/terminal_bench/utils.py | 42 ++++- evalscope/tests/benchmark/test_deep_swe.py | 67 +++++++- .../tests/benchmark/test_terminal_bench.py | 34 ++++ 8 files changed, 379 insertions(+), 39 deletions(-) diff --git a/bash/run.py b/bash/run.py index f4c99d5..73c04e4 100644 --- a/bash/run.py +++ b/bash/run.py @@ -600,6 +600,7 @@ def build_task_config( thinking_max_tokens_scale: float = 1.0, max_tokens_add: int = 0, thinking_budget_tokens: int = None, + truncation_tokens: int = None, ) -> TaskConfig: if run_idx > 0: work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}' @@ -641,6 +642,11 @@ def build_task_config( print(f' max_tokens add: {original_max_tokens} -> {new_max_tokens}') dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) dataset_args.setdefault('shuffle', True) + # Terminal-Bench uses truncation_tokens as a context-limit fallback. + # DeepSWE rejects unknown extra_params, so never inject it there. + if truncation_tokens is not None and dataset_name != 'deep_swe': + extra_params = dataset_args.setdefault('extra_params', {}) + extra_params.setdefault('truncation_tokens', truncation_tokens) if dataset_name in MATH_DATASETS: dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE @@ -668,6 +674,7 @@ def build_task_config( generation_config=generation_config, dataset_args=dataset_args_dict, agent_config=agent_config, + ignore_errors=bool(ds_cfg.get('ignore_errors', False)), eval_batch_size=batch_size, sandbox=SandboxTaskConfig( enabled=True, @@ -1001,6 +1008,7 @@ def main(): thinking_max_tokens_scale=args.thinking_max_tokens_scale, max_tokens_add=args.max_tokens_add, thinking_budget_tokens=args.thinking_budget_tokens, + truncation_tokens=args.truncation_tokens, ) try: run_and_summarize(task_cfg, write_summary_flag, str(model_output_dir), args.model, diff --git a/config/dpv4-int8_nothinking.yaml b/config/dpv4-int8_nothinking.yaml index 0480c73..fdf2cc7 100644 --- a/config/dpv4-int8_nothinking.yaml +++ b/config/dpv4-int8_nothinking.yaml @@ -204,6 +204,8 @@ swe_bench_pro: stream: true max_tokens: 32768 terminal_bench_v2_1: + # 单题 Harbor 异常（如 AgentTimeoutError）不中止整项；超时题不计入分数 + ignore_errors: true generation_config: temperature: 0.0 top_p: 1.0 @@ -211,8 +213,14 @@ terminal_bench_v2_1: max_tokens: 8192 dataset_args: extra_params: - timeout_multiplier: 2.0 - max_turns: 500 + # 任务默认 agent 时限约 900s；倍率 6 → 约 1 小时。绝对秒数用 agent_timeout_sec（勿与 agent_timeout_multiplier 同设） + timeout_multiplier: 6.0 + max_turns: 200 + # 上下窗口：context_limit > generation_config.max_tokens > --truncation-tokens > 100000 + # 必须显式设 context_limit，否则 max_tokens=8192 会被当成上下文，terminus-2 几乎每轮摘要 + context_limit: 65536 + enable_summarize: true + proactive_summarization_threshold: 8000 aa_lcr: generation_config: temperature: 1.0 @@ -233,6 +241,8 @@ deep_swe: max_tokens: 32768 dataset_args: extra_params: + # 官方 Claude 等原生 provider 改 pier_route: native + pier_route: openai_compat pier_model_prefix: openai pier_agent_kwargs: model_class: litellm diff --git a/config/dpv4-int8_thinking.yaml b/config/dpv4-int8_thinking.yaml index 2f37153..e9404d9 100644 --- a/config/dpv4-int8_thinking.yaml +++ b/config/dpv4-int8_thinking.yaml @@ -257,6 +257,7 @@ deep_swe: max_completion_tokens: 64000 dataset_args: extra_params: + pier_route: openai_compat pier_model_prefix: openai pier_agent_kwargs: model_class: litellm @@ -320,7 +321,10 @@ terminal_bench_v2_1: dataset_args: extra_params: timeout_multiplier: 2.0 - max_turns: 500 + max_turns: 200 + context_limit: 65536 + enable_summarize: true + proactive_summarization_threshold: 8000 browsecomp: generation_config: temperature: 1.0 diff --git a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py index 36f9f91..08b7ae8 100644 --- a/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py +++ b/evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py @@ -1,6 +1,8 @@ import uuid +from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.parse import urlparse, urlunparse from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta from evalscope.api.dataset import DatasetDict, Sample, build_dataset_from_records @@ -54,16 +56,117 @@ COMMON_EXTRA_PARAMS = { }, 'pier_model_prefix': { 'type': 'str', - 'description': 'LiteLLM provider prefix prepended when TaskConfig.model has no slash (e.g. openai).', + 'description': ( + 'LiteLLM provider prepended in openai_compat mode (default openai). ' + 'Applied even when the served model id already contains a slash.' + ), 'value': 'openai', }, 'pier_model_name': { 'type': 'str', - 'description': 'Optional full Pier model name provider/model. Overrides TaskConfig.model when set.', + 'description': 'Optional served model id for Pier. Overrides TaskConfig.model, then still normalized by pier_route.', 'value': '', }, + 'pier_route': { + 'type': 'str', + 'description': ( + "openai_compat (default): always use openai/ for LiteLLM. " + "native: pass the model string through (e.g. anthropic/claude-opus-4-8)." + ), + 'value': 'openai_compat', + }, + 'extra_allowed_hosts': { + 'type': 'list', + 'description': ( + 'Optional extra Pier agent allowlist hosts. The hostname/IP from TaskConfig.api_url ' + 'is always added automatically; use this only for additional destinations.' + ), + 'value': [], + }, } +_LOOPBACK_HOSTS = {'127.0.0.1', 'localhost', '::1'} +_HOST_GATEWAY = 'host.docker.internal' +_CHAT_COMPLETIONS_SUFFIXES = ('/chat/completions', '/completions') +_HOST_GATEWAY_COMPOSE = """\ +services: + main: + extra_hosts: + - "host.docker.internal:host-gateway" +""" + + +@dataclass(frozen=True) +class PierApiEndpoint: + """Normalized OpenAI-compatible endpoint derived from TaskConfig.api_url.""" + + base_url: str + allowlist_host: str + uses_host_gateway: bool + + +def resolve_pier_api_endpoint(api_url: str) -> Optional[PierApiEndpoint]: + """Turn an EvalScope api_url into a sandbox-reachable LiteLLM base URL. + + Strips ``/chat/completions``, keeps ``/v1``, rewrites loopback hosts to + ``host.docker.internal`` so Pier's Docker agent can reach the host or a + remote OpenAI-compatible server. Returns None when api_url is empty. + """ + raw = (api_url or '').strip() + if not raw: + return None + parsed = urlparse(raw if '://' in raw else f'http://{raw}') + host = (parsed.hostname or '').strip('[]') + if not host: + return None + + uses_host_gateway = host.lower() in _LOOPBACK_HOSTS + if uses_host_gateway: + host = _HOST_GATEWAY + + path = (parsed.path or '').rstrip('/') + for suffix in _CHAT_COMPLETIONS_SUFFIXES: + if path.endswith(suffix): + path = path[: -len(suffix)].rstrip('/') + break + if not path: + path = '/v1' + + netloc = f'{host}:{parsed.port}' if parsed.port else host + scheme = parsed.scheme or 'http' + base_url = urlunparse((scheme, netloc, path, '', '', '')).rstrip('/') + return PierApiEndpoint(base_url=base_url, allowlist_host=host, uses_host_gateway=uses_host_gateway) + + +PIER_ROUTE_OPENAI_COMPAT = 'openai_compat' +PIER_ROUTE_NATIVE = 'native' + + +def resolve_pier_model_name( + served_name: str, + *, + route: str = PIER_ROUTE_OPENAI_COMPAT, + prefix: str = 'openai', +) -> str: + """Map a served model id to the LiteLLM/Pier ``--model`` string. + + ``openai_compat`` (default) always uses ``{prefix}/``. A slash in + the served id is part of the gateway model name, not a LiteLLM provider + (e.g. ``DeepSeek/DeepSeek-V4-Flash-0731`` -> ``openai/DeepSeek/DeepSeek-V4-Flash-0731``). + ``native`` leaves official provider routes unchanged (``anthropic/...``). + """ + name = (served_name or '').strip() + if not name: + return name + route_key = (route or PIER_ROUTE_OPENAI_COMPAT).strip().lower() + if route_key == PIER_ROUTE_NATIVE: + return name + provider = (prefix or 'openai').strip().strip('/') or 'openai' + marker = f'{provider}/' + if name.startswith(marker) or name.lower().startswith(marker.lower()): + return name + return f'{provider}/{name}' + class DeepSWEAdapter(AgentAdapter): """EvalScope adapter for DeepSWE through Pier Python API jobs.""" @@ -78,6 +181,8 @@ class DeepSWEAdapter(AgentAdapter): self.pier_agent_kwargs = dict(extra_params.get('pier_agent_kwargs') or {}) self.pier_model_prefix = str(extra_params.get('pier_model_prefix') or 'openai') self.pier_model_name = str(extra_params.get('pier_model_name') or '') + self.pier_route = str(extra_params.get('pier_route') or PIER_ROUTE_OPENAI_COMPAT) + self.extra_allowed_hosts = self._as_list(extra_params.get('extra_allowed_hosts') or []) @staticmethod def _as_list(value: Union[str, List[Any], Tuple[Any, ...]]) -> List[str]: @@ -145,6 +250,10 @@ class DeepSWEAdapter(AgentAdapter): task_id = str(sample.metadata['task_id']) pier_kwargs = dict(self.pier_agent_kwargs) pier_kwargs.setdefault('model_class', 'litellm') + endpoint = self._pier_api_endpoint() + environment_kwargs: Dict[str, Any] = {'type': 'docker'} + if endpoint is not None and endpoint.uses_host_gateway: + environment_kwargs['extra_docker_compose'] = [self._host_gateway_compose_path()] config = JobConfig( job_name=f'{task_id[:48].rstrip("_-")}__{uuid.uuid4().hex[:8]}', @@ -161,10 +270,11 @@ class DeepSWEAdapter(AgentAdapter): name='mini-swe-agent', model_name=self._pier_model_name(model), kwargs=pier_kwargs, - env=self._pier_agent_env(), + env=self._pier_agent_env(endpoint), + extra_allowed_hosts=self._pier_allowed_hosts(endpoint), ) ], - environment=EnvironmentConfig(type='docker'), + environment=EnvironmentConfig(**environment_kwargs), verifier=VerifierConfig(env={}), tasks=[TaskConfig(path=Path(sample.metadata['task_path']))], ) @@ -181,10 +291,11 @@ class DeepSWEAdapter(AgentAdapter): def _pier_model_name(self, model: Model) -> str: name = self.pier_model_name or (model.name if model else '') - name = str(name).strip() - if name and '/' not in name: - name = f'{self.pier_model_prefix}/{name}' - return name + return resolve_pier_model_name( + str(name), + route=self.pier_route, + prefix=self.pier_model_prefix, + ) @staticmethod def _plain_secret(value: Any) -> str: @@ -194,18 +305,39 @@ class DeepSWEAdapter(AgentAdapter): value = value.get_secret_value() return str(value).strip() - def _pier_agent_env(self) -> Dict[str, str]: + def _pier_api_endpoint(self) -> Optional[PierApiEndpoint]: + tc = self._task_config + if tc is None: + return None + return resolve_pier_api_endpoint(self._plain_secret(getattr(tc, 'api_url', None))) + + def _pier_allowed_hosts(self, endpoint: Optional[PierApiEndpoint] = None) -> List[str]: + endpoint = self._pier_api_endpoint() if endpoint is None else endpoint + hosts: List[str] = [] + if endpoint is not None: + hosts.append(endpoint.allowlist_host) + hosts.extend(self.extra_allowed_hosts) + return list(dict.fromkeys(host for host in hosts if host)) + + def _host_gateway_compose_path(self) -> Path: + path = Path(self.output_dir) / 'deep_swe_host_gateway.compose.yaml' + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_HOST_GATEWAY_COMPOSE, encoding='utf-8') + return path + + def _pier_agent_env(self, endpoint: Optional[PierApiEndpoint] = None) -> Dict[str, str]: env: Dict[str, str] = { 'LITELLM_LOCAL_MODEL_COST_MAP': 'true', } tc = self._task_config if tc is None: return env - api_url = self._plain_secret(getattr(tc, 'api_url', None)) + endpoint = self._pier_api_endpoint() if endpoint is None else endpoint api_key = self._plain_secret(getattr(tc, 'api_key', None)) - if api_url: - env['OPENAI_API_BASE'] = api_url - env['OPENAI_BASE_URL'] = api_url + if endpoint is not None: + env['OPENAI_API_BASE'] = endpoint.base_url + env['OPENAI_BASE_URL'] = endpoint.base_url if api_key and api_key != 'EMPTY': env['OPENAI_API_KEY'] = api_key env['MSWEA_API_KEY'] = api_key @@ -266,8 +398,10 @@ integrates it through Pier and runs each benchmark sample as one Pier Python API - Dataset defaults to ModelScope `evalscope/deep-swe` - DeepSWE runs through Pier's Docker environment in EvalScope - Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API -- If `TaskConfig.model` has no slash, EvalScope prefixes `pier_model_prefix` (default `openai/`) for Pier/LiteLLM +- `pier_route=openai_compat` (default) always sends LiteLLM `openai/`, including served ids that already contain a slash +- `pier_route=native` passes the model string through for official providers such as `anthropic/...` - API base/key are forwarded as `OPENAI_API_BASE` / `OPENAI_API_KEY` from TaskConfig +- The hostname/IP in `TaskConfig.api_url` is added to Pier `extra_allowed_hosts` automatically; loopback URLs are rewritten to `host.docker.internal` - Optional offline Pier agent install: run `bash/images_load/preload_deep_swe_agent_offline.sh` so sandbox builds COPY local uv/wheels instead of curling GitHub """, dataset_id=DEFAULT_MODELSCOPE_DATASET_ID, diff --git a/evalscope/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py b/evalscope/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py index 019aa04..3435453 100644 --- a/evalscope/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py +++ b/evalscope/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py @@ -47,31 +47,35 @@ except Exception: pass # 把容器内的 apt 源换成清华镜像，避免 tmux/asciinema 安装时 apt-get update 慢/超时。 -# 可以通过环境变量 HARBOR_APT_MIRROR 切换镜像地址。 +# 必须同时改 classic .list 和 Ubuntu 24.04 / Debian 12 的 deb822 .sources， +# 否则 log-summary 这类镜像仍走 deb.debian.org，Packages 索引可能下几百秒。 +# 默认用 http：任务镜像常缺 CA，且实测比 https 更快。可用 HARBOR_APT_MIRROR 覆盖。 try: from harbor.agents.terminus_2.tmux_session import TmuxSession as _TmuxSession - _APT_MIRROR = os.environ.get('HARBOR_APT_MIRROR', 'https://mirrors.tuna.tsinghua.edu.cn') + _APT_MIRROR = os.environ.get('HARBOR_APT_MIRROR', 'http://mirrors.tuna.tsinghua.edu.cn') _ORIG_GET_COMBINED_INSTALL_COMMAND = _TmuxSession._get_combined_install_command def _patched_get_combined_install_command(self, system_info, tools): package_manager = system_info.get('package_manager') if isinstance(system_info, dict) else None if package_manager == 'apt-get': packages = ' '.join(tools) + sed_expr = ( + f"s|https\\?://archive.ubuntu.com/ubuntu|{_APT_MIRROR}/ubuntu|g; " + f"s|https\\?://security.ubuntu.com/ubuntu|{_APT_MIRROR}/ubuntu|g; " + f"s|https\\?://ports.ubuntu.com/ubuntu-ports|{_APT_MIRROR}/ubuntu-ports|g; " + f"s|https\\?://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; " + f"s|https\\?://deb.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; " + f"s|https\\?://deb.debian.org/debian|{_APT_MIRROR}/debian|g" + ) return ( - f"sed -i 's|http://archive.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; " - f"s|https://archive.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; " - f"s|http://security.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; " - f"s|https://security.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; " - f"s|http://ports.ubuntu.com/ubuntu-ports/|{_APT_MIRROR}/ubuntu-ports/|g; " - f"s|https://ports.ubuntu.com/ubuntu-ports/|{_APT_MIRROR}/ubuntu-ports/|g; " - f"s|http://deb.debian.org/debian|{_APT_MIRROR}/debian|g; " - f"s|https://deb.debian.org/debian|{_APT_MIRROR}/debian|g; " - f"s|http://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; " - f"s|https://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g' " - f"/etc/apt/sources.list /etc/apt/sources.list.d/*.list 2>/dev/null; " - f"DEBIAN_FRONTEND=noninteractive apt-get update && " - f"DEBIAN_FRONTEND=noninteractive apt-get install -y {packages}" + 'for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list ' + '/etc/apt/sources.list.d/*.sources; do ' + '[ -f "$f" ] || continue; ' + f"sed -i '{sed_expr}' \"$f\"; " + 'done; ' + 'DEBIAN_FRONTEND=noninteractive apt-get update && ' + f'DEBIAN_FRONTEND=noninteractive apt-get install -y {packages}' ) return _ORIG_GET_COMBINED_INSTALL_COMMAND(self, system_info, tools) @@ -132,6 +136,35 @@ COMMON_EXTRA_PARAMS = { 'description': 'Maximum number of turns for the agent to complete the task.', 'value': 200, }, + 'context_limit': { + 'type': 'int', + 'description': ( + 'Terminus-2 context window for summarization. Priority: context_limit > ' + 'generation_config.max_tokens > truncation_tokens > 100000.' + ), + 'value': None, + }, + 'enable_summarize': { + 'type': 'bool', + 'description': 'Enable terminus-2 context summarization.', + 'value': True, + }, + 'proactive_summarization_threshold': { + 'type': 'int', + 'description': ( + 'Summarize when remaining context tokens fall below this value. ' + 'Set 0 to disable proactive summarization.' + ), + 'value': 8000, + }, + 'truncation_tokens': { + 'type': 'int', + 'description': ( + 'Fallback context window from --truncation-tokens. Used only when ' + 'context_limit and generation_config.max_tokens are unset.' + ), + 'value': None, + }, 'environment_kwargs': { 'type': 'dict', 'description': 'Extra kwargs passed to Harbor EnvironmentConfig. ' @@ -192,6 +225,20 @@ class _TerminalBenchBase(AgentAdapter): ) self.max_turns = self.extra_params.get('max_turns', 200) self.environment_kwargs = self.extra_params.get('environment_kwargs', {}) + self.context_limit = self.extra_params.get('context_limit') + self.enable_summarize = self.extra_params.get('enable_summarize', True) + self.proactive_summarization_threshold = self.extra_params.get( + 'proactive_summarization_threshold', 8000 + ) + + def _resolve_context_limit(self, model: Model) -> int: + from .utils import resolve_terminus_context_limit + + return resolve_terminus_context_limit( + context_limit=self.context_limit, + max_tokens=getattr(getattr(model, 'config', None), 'max_tokens', None), + truncation_tokens=self.extra_params.get('truncation_tokens'), + ) def load(self): _validate_environment_requirements(self.environment_type) @@ -242,8 +289,8 @@ class _TerminalBenchBase(AgentAdapter): agent_kwargs.update( { 'parser_name': 'json', - 'enable_summarize': True, - 'proactive_summarization_threshold': 8000, + 'enable_summarize': bool(self.enable_summarize), + 'proactive_summarization_threshold': int(self.proactive_summarization_threshold), 'collect_rollout_details': False, } ) @@ -268,7 +315,9 @@ class _TerminalBenchBase(AgentAdapter): ) try: - harbor_llm = HarborLLM(model=model) if self.agent_name == 'terminus-2' else None + harbor_llm = None + if self.agent_name == 'terminus-2': + harbor_llm = HarborLLM(model=model, context_limit=self._resolve_context_limit(model)) async def _run_trial(): trial = await Trial.create(trial_config) diff --git a/evalscope/evalscope/benchmarks/terminal_bench/utils.py b/evalscope/evalscope/benchmarks/terminal_bench/utils.py index fe3719f..4efb170 100644 --- a/evalscope/evalscope/benchmarks/terminal_bench/utils.py +++ b/evalscope/evalscope/benchmarks/terminal_bench/utils.py @@ -1,5 +1,5 @@ import asyncio -from typing import List +from typing import Any, List, Optional from harbor.llms.base import BaseLLM, LLMResponse, UsageInfo from pydantic import BaseModel, ConfigDict, PrivateAttr @@ -9,17 +9,53 @@ from evalscope.api.messages.perf_metrics import PerformanceMetrics from evalscope.api.model.model import Model from evalscope.models.utils.openai import openai_chat_choices +HARBOR_DEFAULT_CONTEXT_LIMIT = 100_000 + + +def _positive_int(value: Any) -> Optional[int]: + if value in (None, ''): + return None + try: + parsed = int(value) + except (TypeError, ValueError): + return None + return parsed if parsed > 0 else None + + +def resolve_terminus_context_limit( + *, + context_limit: Any = None, + max_tokens: Any = None, + truncation_tokens: Any = None, + default: int = HARBOR_DEFAULT_CONTEXT_LIMIT, +) -> int: + """Context window for terminus-2 summarization. + + Priority: extra_params.context_limit > generation_config.max_tokens > + truncation-tokens > Harbor default 100000. + """ + for value in (context_limit, max_tokens, truncation_tokens): + parsed = _positive_int(value) + if parsed is not None: + return parsed + return default + class HarborLLM(BaseModel, BaseLLM): """A mock LLM that simulates sandboxed code execution.""" model_config = ConfigDict(arbitrary_types_allowed=True) _model: Model = PrivateAttr() + _context_limit: int = PrivateAttr() _perf_metrics: List[PerformanceMetrics] = PrivateAttr(default_factory=list) - def __init__(self, model: Model, **kwargs): + def __init__(self, model: Model, context_limit: Optional[int] = None, **kwargs): super().__init__(**kwargs) self._model = model + self._context_limit = resolve_terminus_context_limit( + context_limit=context_limit, + max_tokens=getattr(getattr(model, 'config', None), 'max_tokens', None), + ) @property def model(self): @@ -58,7 +94,7 @@ class HarborLLM(BaseModel, BaseLLM): ) def get_model_context_limit(self): - return self._model.config.max_tokens or 100_000 + return self._context_limit def get_model_output_limit(self): return self._model.config.max_tokens or 16384 diff --git a/evalscope/tests/benchmark/test_deep_swe.py b/evalscope/tests/benchmark/test_deep_swe.py index 82038c8..633419c 100644 --- a/evalscope/tests/benchmark/test_deep_swe.py +++ b/evalscope/tests/benchmark/test_deep_swe.py @@ -11,7 +11,12 @@ import pytest from evalscope.api.dataset import Sample from evalscope.api.evaluator import TaskState from evalscope.api.registry import get_benchmark -from evalscope.benchmarks.deep_swe.deep_swe_adapter import DEFAULT_MODELSCOPE_DATASET_ID, DeepSWEAdapter +from evalscope.benchmarks.deep_swe.deep_swe_adapter import ( + DEFAULT_MODELSCOPE_DATASET_ID, + DeepSWEAdapter, + resolve_pier_api_endpoint, + resolve_pier_model_name, +) from evalscope.benchmarks.deep_swe.utils import artifact_path, build_score_metadata, parse_timestamp from evalscope.config import TaskConfig @@ -297,6 +302,62 @@ def install_fake_pier(monkeypatch: Any, captured: Dict[str, Any], result: Dict[s monkeypatch.setattr('evalscope.benchmarks.deep_swe.deep_swe_adapter.check_import', lambda *args, **kwargs: True) +def test_resolve_pier_api_endpoint_from_local_intranet_and_gateway_urls() -> None: + local = resolve_pier_api_endpoint('http://127.0.0.1:8004/v1') + assert local is not None + assert local.base_url == 'http://host.docker.internal:8004/v1' + assert local.allowlist_host == 'host.docker.internal' + assert local.uses_host_gateway is True + + intranet = resolve_pier_api_endpoint('http://10.1.2.3:8000/v1') + assert intranet is not None + assert intranet.base_url == 'http://10.1.2.3:8000/v1' + assert intranet.allowlist_host == '10.1.2.3' + assert intranet.uses_host_gateway is False + + gateway = resolve_pier_api_endpoint('https://api.vectron.meta-stone.com/v1/chat/completions') + assert gateway is not None + assert gateway.base_url == 'https://api.vectron.meta-stone.com/v1' + assert gateway.allowlist_host == 'api.vectron.meta-stone.com' + assert gateway.uses_host_gateway is False + + +def test_resolve_pier_model_name_openai_compat_and_native() -> None: + assert resolve_pier_model_name('DeepSeek/DeepSeek-V4-Flash-0731') == ( + 'openai/DeepSeek/DeepSeek-V4-Flash-0731' + ) + assert resolve_pier_model_name('Qwen3-VL-8B-Instruct') == 'openai/Qwen3-VL-8B-Instruct' + assert resolve_pier_model_name('openai/gpt-5.5') == 'openai/gpt-5.5' + assert resolve_pier_model_name( + 'anthropic/claude-opus-4-8', + route='native', + ) == 'anthropic/claude-opus-4-8' + + +def test_pier_model_name_uses_openai_compat_by_default(tmp_path: Path) -> None: + class SlashModel: + name = 'DeepSeek/DeepSeek-V4-Flash-0731' + + adapter = make_adapter(tmp_path) + assert adapter._pier_model_name(SlashModel()) == 'openai/DeepSeek/DeepSeek-V4-Flash-0731' + + native = make_adapter(tmp_path, pier_route='native') + assert native._pier_model_name(SlashModel()) == 'DeepSeek/DeepSeek-V4-Flash-0731' + + +def test_pier_allowed_hosts_come_from_api_url(tmp_path: Path) -> None: + cfg = TaskConfig( + datasets=['deep_swe'], + api_url='https://api.vectron.meta-stone.com/v1/chat/completions', + api_key='sk-test', + work_dir=str(tmp_path / 'outputs'), + dataset_args={'deep_swe': {'extra_params': {'extra_allowed_hosts': ['10.9.9.9']}}}, + ) + adapter = get_benchmark('deep_swe', cfg) + assert isinstance(adapter, DeepSWEAdapter) + assert adapter._pier_allowed_hosts() == ['api.vectron.meta-stone.com', '10.9.9.9'] + + def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) -> None: captured: Dict[str, Any] = {} result = { @@ -313,6 +374,7 @@ def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) - task_path = tmp_path / 'tasks' / 'task-a' task_path.mkdir(parents=True) adapter = make_adapter(tmp_path, pier_agent_kwargs={'model_class': 'litellm'}) + adapter._task_config.api_url = 'http://10.1.2.3:8000/v1' sample = Sample(input='', metadata={'task_id': 'task-a', 'task_path': str(task_path)}) result_dict = adapter._run_pier_job(MockModel(), sample) @@ -321,7 +383,10 @@ def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) - assert captured['config'].tasks[0].path == task_path assert getattr(captured['config'].tasks[0], 'source', None) is None assert captured['config'].agents[0].kwargs == {'model_class': 'litellm'} + assert captured['config'].agents[0].model_name == 'openai/mock-model' + assert captured['config'].agents[0].extra_allowed_hosts == ['10.1.2.3'] assert captured['config'].environment.type == 'docker' + assert getattr(captured['config'].environment, 'extra_docker_compose', None) in (None, []) @pytest.mark.skipif( diff --git a/evalscope/tests/benchmark/test_terminal_bench.py b/evalscope/tests/benchmark/test_terminal_bench.py index 5728b39..10407eb 100644 --- a/evalscope/tests/benchmark/test_terminal_bench.py +++ b/evalscope/tests/benchmark/test_terminal_bench.py @@ -8,6 +8,7 @@ import pytest from evalscope.api.messages.perf_metrics import PerformanceMetrics from evalscope.api.metric import Score from evalscope.benchmarks.terminal_bench.terminal_bench_adapter import _phase_timeout_options, _TerminalBenchBase +from evalscope.benchmarks.terminal_bench.utils import HARBOR_DEFAULT_CONTEXT_LIMIT, resolve_terminus_context_limit TRIAL_URI = 'file:///tmp/terminal-bench-trial' @@ -145,3 +146,36 @@ def test_terminal_bench_trace_preserves_request_perf_metrics(tmp_path) -> None: assert messages[0].perf_metrics.latency == 1.0 assert messages[0].perf_metrics.input_tokens == 3 + + +@pytest.mark.parametrize( + ('kwargs', 'expected'), + [ + ({'context_limit': 65536, 'max_tokens': 8192, 'truncation_tokens': 131072}, 65536), + ({'max_tokens': 8192, 'truncation_tokens': 131072}, 8192), + ({'truncation_tokens': 65536}, 65536), + ({}, HARBOR_DEFAULT_CONTEXT_LIMIT), + ({'context_limit': 0, 'max_tokens': 8192, 'truncation_tokens': 65536}, 8192), + ({'context_limit': '', 'max_tokens': None, 'truncation_tokens': 65536}, 65536), + ], +) +def test_terminus_context_limit_priority(kwargs, expected) -> None: + assert resolve_terminus_context_limit(**kwargs) == expected + + +def test_adapter_resolves_context_limit_from_extra_params() -> None: + adapter = object.__new__(_TerminalBenchBase) + adapter.context_limit = 65536 + adapter.extra_params = {'truncation_tokens': 131072} + model = SimpleNamespace(config=SimpleNamespace(max_tokens=8192)) + + assert adapter._resolve_context_limit(model) == 65536 + + adapter.context_limit = None + assert adapter._resolve_context_limit(model) == 8192 + + model.config.max_tokens = None + assert adapter._resolve_context_limit(model) == 131072 + + adapter.extra_params = {} + assert adapter._resolve_context_limit(model) == HARBOR_DEFAULT_CONTEXT_LIMIT