Fix DeepSWE Pier routing and Terminal-Bench context/apt setup.

Keep generation max_tokens from being used as terminus-2 context, rewrite Debian/Ubuntu .sources to the Tsinghua mirror, and skip injecting truncation_tokens into DeepSWE extra_params.

Co-authored-by: Cursor <cursoragent@cursor.com>

bash/run.py

@@ -600,6 +600,7 @@ def build_task_config(
     thinking_max_tokens_scale: float = 1.0,
     max_tokens_add: int = 0,
     thinking_budget_tokens: int = None,
+    truncation_tokens: int = None,
 ) -> TaskConfig:
     if run_idx > 0:
         work_dir = Path(output_dir) / dataset_name / f'seed_{seed}_run_{run_idx}'
@@ -641,6 +642,11 @@ def build_task_config(
         print(f'   max_tokens add: {original_max_tokens} -> {new_max_tokens}')
     dataset_args = deepcopy(ds_cfg.get('dataset_args', {}))
     dataset_args.setdefault('shuffle', True)
+    # Terminal-Bench uses truncation_tokens as a context-limit fallback.
+    # DeepSWE rejects unknown extra_params, so never inject it there.
+    if truncation_tokens is not None and dataset_name != 'deep_swe':
+        extra_params = dataset_args.setdefault('extra_params', {})
+        extra_params.setdefault('truncation_tokens', truncation_tokens)
 
     if dataset_name in MATH_DATASETS:
         dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE
@@ -668,6 +674,7 @@ def build_task_config(
         generation_config=generation_config,
         dataset_args=dataset_args_dict,
         agent_config=agent_config,
+        ignore_errors=bool(ds_cfg.get('ignore_errors', False)),
         eval_batch_size=batch_size,
         sandbox=SandboxTaskConfig(
             enabled=True,
@@ -1001,6 +1008,7 @@ def main():
             thinking_max_tokens_scale=args.thinking_max_tokens_scale,
             max_tokens_add=args.max_tokens_add,
             thinking_budget_tokens=args.thinking_budget_tokens,
+            truncation_tokens=args.truncation_tokens,
         )
         try:
             run_and_summarize(task_cfg, write_summary_flag, str(model_output_dir), args.model,

config/dpv4-int8_nothinking.yaml

@@ -204,6 +204,8 @@ swe_bench_pro:
     stream: true
     max_tokens: 32768
 terminal_bench_v2_1:
+  # 单题 Harbor 异常（如 AgentTimeoutError）不中止整项；超时题不计入分数
+  ignore_errors: true
   generation_config:
     temperature: 0.0
     top_p: 1.0
@@ -211,8 +213,14 @@ terminal_bench_v2_1:
     max_tokens: 8192
   dataset_args:
     extra_params:
-      timeout_multiplier: 2.0
-      max_turns: 500
+      # 任务默认 agent 时限约 900s；倍率 6 → 约 1 小时。绝对秒数用 agent_timeout_sec（勿与 agent_timeout_multiplier 同设）
+      timeout_multiplier: 6.0
+      max_turns: 200
+      # 上下窗口：context_limit > generation_config.max_tokens > --truncation-tokens > 100000
+      # 必须显式设 context_limit，否则 max_tokens=8192 会被当成上下文，terminus-2 几乎每轮摘要
+      context_limit: 65536
+      enable_summarize: true
+      proactive_summarization_threshold: 8000
 aa_lcr:
   generation_config:
     temperature: 1.0
@@ -233,6 +241,8 @@ deep_swe:
     max_tokens: 32768
   dataset_args:
     extra_params:
+      # 官方 Claude 等原生 provider 改 pier_route: native
+      pier_route: openai_compat
       pier_model_prefix: openai
       pier_agent_kwargs:
         model_class: litellm

config/dpv4-int8_thinking.yaml

@@ -257,6 +257,7 @@ deep_swe:
     max_completion_tokens: 64000
   dataset_args:
     extra_params:
+      pier_route: openai_compat
       pier_model_prefix: openai
       pier_agent_kwargs:
         model_class: litellm
@@ -320,7 +321,10 @@ terminal_bench_v2_1:
   dataset_args:
     extra_params:
       timeout_multiplier: 2.0
-      max_turns: 500
+      max_turns: 200
+      context_limit: 65536
+      enable_summarize: true
+      proactive_summarization_threshold: 8000
 browsecomp:
   generation_config:
     temperature: 1.0

evalscope/evalscope/benchmarks/deep_swe/deep_swe_adapter.py

@@ -1,6 +1,8 @@
 import uuid
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlparse, urlunparse
 
 from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
 from evalscope.api.dataset import DatasetDict, Sample, build_dataset_from_records
@@ -54,16 +56,117 @@ COMMON_EXTRA_PARAMS = {
     },
     'pier_model_prefix': {
         'type': 'str',
-        'description': 'LiteLLM provider prefix prepended when TaskConfig.model has no slash (e.g. openai).',
+        'description': (
+            'LiteLLM provider prepended in openai_compat mode (default openai). '
+            'Applied even when the served model id already contains a slash.'
+        ),
         'value': 'openai',
     },
     'pier_model_name': {
         'type': 'str',
-        'description': 'Optional full Pier model name provider/model. Overrides TaskConfig.model when set.',
+        'description': 'Optional served model id for Pier. Overrides TaskConfig.model, then still normalized by pier_route.',
         'value': '',
     },
+    'pier_route': {
+        'type': 'str',
+        'description': (
+            "openai_compat (default): always use openai/<served-id> for LiteLLM. "
+            "native: pass the model string through (e.g. anthropic/claude-opus-4-8)."
+        ),
+        'value': 'openai_compat',
+    },
+    'extra_allowed_hosts': {
+        'type': 'list',
+        'description': (
+            'Optional extra Pier agent allowlist hosts. The hostname/IP from TaskConfig.api_url '
+            'is always added automatically; use this only for additional destinations.'
+        ),
+        'value': [],
+    },
 }
 
+_LOOPBACK_HOSTS = {'127.0.0.1', 'localhost', '::1'}
+_HOST_GATEWAY = 'host.docker.internal'
+_CHAT_COMPLETIONS_SUFFIXES = ('/chat/completions', '/completions')
+_HOST_GATEWAY_COMPOSE = """\
+services:
+  main:
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+"""
+
+
+@dataclass(frozen=True)
+class PierApiEndpoint:
+    """Normalized OpenAI-compatible endpoint derived from TaskConfig.api_url."""
+
+    base_url: str
+    allowlist_host: str
+    uses_host_gateway: bool
+
+
+def resolve_pier_api_endpoint(api_url: str) -> Optional[PierApiEndpoint]:
+    """Turn an EvalScope api_url into a sandbox-reachable LiteLLM base URL.
+
+    Strips ``/chat/completions``, keeps ``/v1``, rewrites loopback hosts to
+    ``host.docker.internal`` so Pier's Docker agent can reach the host or a
+    remote OpenAI-compatible server. Returns None when api_url is empty.
+    """
+    raw = (api_url or '').strip()
+    if not raw:
+        return None
+    parsed = urlparse(raw if '://' in raw else f'http://{raw}')
+    host = (parsed.hostname or '').strip('[]')
+    if not host:
+        return None
+
+    uses_host_gateway = host.lower() in _LOOPBACK_HOSTS
+    if uses_host_gateway:
+        host = _HOST_GATEWAY
+
+    path = (parsed.path or '').rstrip('/')
+    for suffix in _CHAT_COMPLETIONS_SUFFIXES:
+        if path.endswith(suffix):
+            path = path[: -len(suffix)].rstrip('/')
+            break
+    if not path:
+        path = '/v1'
+
+    netloc = f'{host}:{parsed.port}' if parsed.port else host
+    scheme = parsed.scheme or 'http'
+    base_url = urlunparse((scheme, netloc, path, '', '', '')).rstrip('/')
+    return PierApiEndpoint(base_url=base_url, allowlist_host=host, uses_host_gateway=uses_host_gateway)
+
+
+PIER_ROUTE_OPENAI_COMPAT = 'openai_compat'
+PIER_ROUTE_NATIVE = 'native'
+
+
+def resolve_pier_model_name(
+    served_name: str,
+    *,
+    route: str = PIER_ROUTE_OPENAI_COMPAT,
+    prefix: str = 'openai',
+) -> str:
+    """Map a served model id to the LiteLLM/Pier ``--model`` string.
+
+    ``openai_compat`` (default) always uses ``{prefix}/<served-id>``. A slash in
+    the served id is part of the gateway model name, not a LiteLLM provider
+    (e.g. ``DeepSeek/DeepSeek-V4-Flash-0731`` -> ``openai/DeepSeek/DeepSeek-V4-Flash-0731``).
+    ``native`` leaves official provider routes unchanged (``anthropic/...``).
+    """
+    name = (served_name or '').strip()
+    if not name:
+        return name
+    route_key = (route or PIER_ROUTE_OPENAI_COMPAT).strip().lower()
+    if route_key == PIER_ROUTE_NATIVE:
+        return name
+    provider = (prefix or 'openai').strip().strip('/') or 'openai'
+    marker = f'{provider}/'
+    if name.startswith(marker) or name.lower().startswith(marker.lower()):
+        return name
+    return f'{provider}/{name}'
+
 
 class DeepSWEAdapter(AgentAdapter):
     """EvalScope adapter for DeepSWE through Pier Python API jobs."""
@@ -78,6 +181,8 @@ class DeepSWEAdapter(AgentAdapter):
         self.pier_agent_kwargs = dict(extra_params.get('pier_agent_kwargs') or {})
         self.pier_model_prefix = str(extra_params.get('pier_model_prefix') or 'openai')
         self.pier_model_name = str(extra_params.get('pier_model_name') or '')
+        self.pier_route = str(extra_params.get('pier_route') or PIER_ROUTE_OPENAI_COMPAT)
+        self.extra_allowed_hosts = self._as_list(extra_params.get('extra_allowed_hosts') or [])
 
     @staticmethod
     def _as_list(value: Union[str, List[Any], Tuple[Any, ...]]) -> List[str]:
@@ -145,6 +250,10 @@ class DeepSWEAdapter(AgentAdapter):
         task_id = str(sample.metadata['task_id'])
         pier_kwargs = dict(self.pier_agent_kwargs)
         pier_kwargs.setdefault('model_class', 'litellm')
+        endpoint = self._pier_api_endpoint()
+        environment_kwargs: Dict[str, Any] = {'type': 'docker'}
+        if endpoint is not None and endpoint.uses_host_gateway:
+            environment_kwargs['extra_docker_compose'] = [self._host_gateway_compose_path()]
 
         config = JobConfig(
             job_name=f'{task_id[:48].rstrip("_-")}__{uuid.uuid4().hex[:8]}',
@@ -161,10 +270,11 @@ class DeepSWEAdapter(AgentAdapter):
                     name='mini-swe-agent',
                     model_name=self._pier_model_name(model),
                     kwargs=pier_kwargs,
-                    env=self._pier_agent_env(),
+                    env=self._pier_agent_env(endpoint),
+                    extra_allowed_hosts=self._pier_allowed_hosts(endpoint),
                 )
             ],
-            environment=EnvironmentConfig(type='docker'),
+            environment=EnvironmentConfig(**environment_kwargs),
             verifier=VerifierConfig(env={}),
             tasks=[TaskConfig(path=Path(sample.metadata['task_path']))],
         )
@@ -181,10 +291,11 @@ class DeepSWEAdapter(AgentAdapter):
 
     def _pier_model_name(self, model: Model) -> str:
         name = self.pier_model_name or (model.name if model else '')
-        name = str(name).strip()
-        if name and '/' not in name:
-            name = f'{self.pier_model_prefix}/{name}'
-        return name
+        return resolve_pier_model_name(
+            str(name),
+            route=self.pier_route,
+            prefix=self.pier_model_prefix,
+        )
 
     @staticmethod
     def _plain_secret(value: Any) -> str:
@@ -194,18 +305,39 @@ class DeepSWEAdapter(AgentAdapter):
             value = value.get_secret_value()
         return str(value).strip()
 
-    def _pier_agent_env(self) -> Dict[str, str]:
+    def _pier_api_endpoint(self) -> Optional[PierApiEndpoint]:
+        tc = self._task_config
+        if tc is None:
+            return None
+        return resolve_pier_api_endpoint(self._plain_secret(getattr(tc, 'api_url', None)))
+
+    def _pier_allowed_hosts(self, endpoint: Optional[PierApiEndpoint] = None) -> List[str]:
+        endpoint = self._pier_api_endpoint() if endpoint is None else endpoint
+        hosts: List[str] = []
+        if endpoint is not None:
+            hosts.append(endpoint.allowlist_host)
+        hosts.extend(self.extra_allowed_hosts)
+        return list(dict.fromkeys(host for host in hosts if host))
+
+    def _host_gateway_compose_path(self) -> Path:
+        path = Path(self.output_dir) / 'deep_swe_host_gateway.compose.yaml'
+        if not path.exists():
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(_HOST_GATEWAY_COMPOSE, encoding='utf-8')
+        return path
+
+    def _pier_agent_env(self, endpoint: Optional[PierApiEndpoint] = None) -> Dict[str, str]:
         env: Dict[str, str] = {
             'LITELLM_LOCAL_MODEL_COST_MAP': 'true',
         }
         tc = self._task_config
         if tc is None:
             return env
-        api_url = self._plain_secret(getattr(tc, 'api_url', None))
+        endpoint = self._pier_api_endpoint() if endpoint is None else endpoint
         api_key = self._plain_secret(getattr(tc, 'api_key', None))
-        if api_url:
-            env['OPENAI_API_BASE'] = api_url
-            env['OPENAI_BASE_URL'] = api_url
+        if endpoint is not None:
+            env['OPENAI_API_BASE'] = endpoint.base_url
+            env['OPENAI_BASE_URL'] = endpoint.base_url
         if api_key and api_key != 'EMPTY':
             env['OPENAI_API_KEY'] = api_key
             env['MSWEA_API_KEY'] = api_key
@@ -266,8 +398,10 @@ integrates it through Pier and runs each benchmark sample as one Pier Python API
 - Dataset defaults to ModelScope `evalscope/deep-swe`
 - DeepSWE runs through Pier's Docker environment in EvalScope
 - Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API
-- If `TaskConfig.model` has no slash, EvalScope prefixes `pier_model_prefix` (default `openai/`) for Pier/LiteLLM
+- `pier_route=openai_compat` (default) always sends LiteLLM `openai/<served-id>`, including served ids that already contain a slash
+- `pier_route=native` passes the model string through for official providers such as `anthropic/...`
 - API base/key are forwarded as `OPENAI_API_BASE` / `OPENAI_API_KEY` from TaskConfig
+- The hostname/IP in `TaskConfig.api_url` is added to Pier `extra_allowed_hosts` automatically; loopback URLs are rewritten to `host.docker.internal`
 - Optional offline Pier agent install: run `bash/images_load/preload_deep_swe_agent_offline.sh` so sandbox builds COPY local uv/wheels instead of curling GitHub
 """,
         dataset_id=DEFAULT_MODELSCOPE_DATASET_ID,

evalscope/evalscope/benchmarks/terminal_bench/terminal_bench_adapter.py

@@ -47,31 +47,35 @@ except Exception:
     pass
 
 # 把容器内的 apt 源换成清华镜像，避免 tmux/asciinema 安装时 apt-get update 慢/超时。
-# 可以通过环境变量 HARBOR_APT_MIRROR 切换镜像地址。
+# 必须同时改 classic .list 和 Ubuntu 24.04 / Debian 12 的 deb822 .sources，
+# 否则 log-summary 这类镜像仍走 deb.debian.org，Packages 索引可能下几百秒。
+# 默认用 http：任务镜像常缺 CA，且实测比 https 更快。可用 HARBOR_APT_MIRROR 覆盖。
 try:
     from harbor.agents.terminus_2.tmux_session import TmuxSession as _TmuxSession
 
-    _APT_MIRROR = os.environ.get('HARBOR_APT_MIRROR', 'https://mirrors.tuna.tsinghua.edu.cn')
+    _APT_MIRROR = os.environ.get('HARBOR_APT_MIRROR', 'http://mirrors.tuna.tsinghua.edu.cn')
     _ORIG_GET_COMBINED_INSTALL_COMMAND = _TmuxSession._get_combined_install_command
 
     def _patched_get_combined_install_command(self, system_info, tools):
         package_manager = system_info.get('package_manager') if isinstance(system_info, dict) else None
         if package_manager == 'apt-get':
             packages = ' '.join(tools)
+            sed_expr = (
+                f"s|https\\?://archive.ubuntu.com/ubuntu|{_APT_MIRROR}/ubuntu|g; "
+                f"s|https\\?://security.ubuntu.com/ubuntu|{_APT_MIRROR}/ubuntu|g; "
+                f"s|https\\?://ports.ubuntu.com/ubuntu-ports|{_APT_MIRROR}/ubuntu-ports|g; "
+                f"s|https\\?://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; "
+                f"s|https\\?://deb.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; "
+                f"s|https\\?://deb.debian.org/debian|{_APT_MIRROR}/debian|g"
+            )
             return (
-                f"sed -i 's|http://archive.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; "
-                f"s|https://archive.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; "
-                f"s|http://security.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; "
-                f"s|https://security.ubuntu.com/ubuntu/|{_APT_MIRROR}/ubuntu/|g; "
-                f"s|http://ports.ubuntu.com/ubuntu-ports/|{_APT_MIRROR}/ubuntu-ports/|g; "
-                f"s|https://ports.ubuntu.com/ubuntu-ports/|{_APT_MIRROR}/ubuntu-ports/|g; "
-                f"s|http://deb.debian.org/debian|{_APT_MIRROR}/debian|g; "
-                f"s|https://deb.debian.org/debian|{_APT_MIRROR}/debian|g; "
-                f"s|http://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g; "
-                f"s|https://security.debian.org/debian-security|{_APT_MIRROR}/debian-security|g' "
-                f"/etc/apt/sources.list /etc/apt/sources.list.d/*.list 2>/dev/null; "
-                f"DEBIAN_FRONTEND=noninteractive apt-get update && "
-                f"DEBIAN_FRONTEND=noninteractive apt-get install -y {packages}"
+                'for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list '
+                '/etc/apt/sources.list.d/*.sources; do '
+                '[ -f "$f" ] || continue; '
+                f"sed -i '{sed_expr}' \"$f\"; "
+                'done; '
+                'DEBIAN_FRONTEND=noninteractive apt-get update && '
+                f'DEBIAN_FRONTEND=noninteractive apt-get install -y {packages}'
             )
         return _ORIG_GET_COMBINED_INSTALL_COMMAND(self, system_info, tools)
 
@@ -132,6 +136,35 @@ COMMON_EXTRA_PARAMS = {
         'description': 'Maximum number of turns for the agent to complete the task.',
         'value': 200,
     },
+    'context_limit': {
+        'type': 'int',
+        'description': (
+            'Terminus-2 context window for summarization. Priority: context_limit > '
+            'generation_config.max_tokens > truncation_tokens > 100000.'
+        ),
+        'value': None,
+    },
+    'enable_summarize': {
+        'type': 'bool',
+        'description': 'Enable terminus-2 context summarization.',
+        'value': True,
+    },
+    'proactive_summarization_threshold': {
+        'type': 'int',
+        'description': (
+            'Summarize when remaining context tokens fall below this value. '
+            'Set 0 to disable proactive summarization.'
+        ),
+        'value': 8000,
+    },
+    'truncation_tokens': {
+        'type': 'int',
+        'description': (
+            'Fallback context window from --truncation-tokens. Used only when '
+            'context_limit and generation_config.max_tokens are unset.'
+        ),
+        'value': None,
+    },
     'environment_kwargs': {
         'type': 'dict',
         'description': 'Extra kwargs passed to Harbor EnvironmentConfig. '
@@ -192,6 +225,20 @@ class _TerminalBenchBase(AgentAdapter):
         )
         self.max_turns = self.extra_params.get('max_turns', 200)
         self.environment_kwargs = self.extra_params.get('environment_kwargs', {})
+        self.context_limit = self.extra_params.get('context_limit')
+        self.enable_summarize = self.extra_params.get('enable_summarize', True)
+        self.proactive_summarization_threshold = self.extra_params.get(
+            'proactive_summarization_threshold', 8000
+        )
+
+    def _resolve_context_limit(self, model: Model) -> int:
+        from .utils import resolve_terminus_context_limit
+
+        return resolve_terminus_context_limit(
+            context_limit=self.context_limit,
+            max_tokens=getattr(getattr(model, 'config', None), 'max_tokens', None),
+            truncation_tokens=self.extra_params.get('truncation_tokens'),
+        )
 
     def load(self):
         _validate_environment_requirements(self.environment_type)
@@ -242,8 +289,8 @@ class _TerminalBenchBase(AgentAdapter):
             agent_kwargs.update(
                 {
                     'parser_name': 'json',
-                    'enable_summarize': True,
-                    'proactive_summarization_threshold': 8000,
+                    'enable_summarize': bool(self.enable_summarize),
+                    'proactive_summarization_threshold': int(self.proactive_summarization_threshold),
                     'collect_rollout_details': False,
                 }
             )
@@ -268,7 +315,9 @@ class _TerminalBenchBase(AgentAdapter):
         )
 
         try:
-            harbor_llm = HarborLLM(model=model) if self.agent_name == 'terminus-2' else None
+            harbor_llm = None
+            if self.agent_name == 'terminus-2':
+                harbor_llm = HarborLLM(model=model, context_limit=self._resolve_context_limit(model))
 
             async def _run_trial():
                 trial = await Trial.create(trial_config)

evalscope/evalscope/benchmarks/terminal_bench/utils.py

@@ -1,5 +1,5 @@
 import asyncio
-from typing import List
+from typing import Any, List, Optional
 
 from harbor.llms.base import BaseLLM, LLMResponse, UsageInfo
 from pydantic import BaseModel, ConfigDict, PrivateAttr
@@ -9,17 +9,53 @@ from evalscope.api.messages.perf_metrics import PerformanceMetrics
 from evalscope.api.model.model import Model
 from evalscope.models.utils.openai import openai_chat_choices
 
+HARBOR_DEFAULT_CONTEXT_LIMIT = 100_000
+
+
+def _positive_int(value: Any) -> Optional[int]:
+    if value in (None, ''):
+        return None
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def resolve_terminus_context_limit(
+    *,
+    context_limit: Any = None,
+    max_tokens: Any = None,
+    truncation_tokens: Any = None,
+    default: int = HARBOR_DEFAULT_CONTEXT_LIMIT,
+) -> int:
+    """Context window for terminus-2 summarization.
+
+    Priority: extra_params.context_limit > generation_config.max_tokens >
+    truncation-tokens > Harbor default 100000.
+    """
+    for value in (context_limit, max_tokens, truncation_tokens):
+        parsed = _positive_int(value)
+        if parsed is not None:
+            return parsed
+    return default
+
 
 class HarborLLM(BaseModel, BaseLLM):
     """A mock LLM that simulates sandboxed code execution."""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
     _model: Model = PrivateAttr()
+    _context_limit: int = PrivateAttr()
     _perf_metrics: List[PerformanceMetrics] = PrivateAttr(default_factory=list)
 
-    def __init__(self, model: Model, **kwargs):
+    def __init__(self, model: Model, context_limit: Optional[int] = None, **kwargs):
         super().__init__(**kwargs)
         self._model = model
+        self._context_limit = resolve_terminus_context_limit(
+            context_limit=context_limit,
+            max_tokens=getattr(getattr(model, 'config', None), 'max_tokens', None),
+        )
 
     @property
     def model(self):
@@ -58,7 +94,7 @@ class HarborLLM(BaseModel, BaseLLM):
         )
 
     def get_model_context_limit(self):
-        return self._model.config.max_tokens or 100_000
+        return self._context_limit
 
     def get_model_output_limit(self):
         return self._model.config.max_tokens or 16384

evalscope/tests/benchmark/test_deep_swe.py

@@ -11,7 +11,12 @@ import pytest
 from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.registry import get_benchmark
-from evalscope.benchmarks.deep_swe.deep_swe_adapter import DEFAULT_MODELSCOPE_DATASET_ID, DeepSWEAdapter
+from evalscope.benchmarks.deep_swe.deep_swe_adapter import (
+    DEFAULT_MODELSCOPE_DATASET_ID,
+    DeepSWEAdapter,
+    resolve_pier_api_endpoint,
+    resolve_pier_model_name,
+)
 from evalscope.benchmarks.deep_swe.utils import artifact_path, build_score_metadata, parse_timestamp
 from evalscope.config import TaskConfig
 
@@ -297,6 +302,62 @@ def install_fake_pier(monkeypatch: Any, captured: Dict[str, Any], result: Dict[s
     monkeypatch.setattr('evalscope.benchmarks.deep_swe.deep_swe_adapter.check_import', lambda *args, **kwargs: True)
 
 
+def test_resolve_pier_api_endpoint_from_local_intranet_and_gateway_urls() -> None:
+    local = resolve_pier_api_endpoint('http://127.0.0.1:8004/v1')
+    assert local is not None
+    assert local.base_url == 'http://host.docker.internal:8004/v1'
+    assert local.allowlist_host == 'host.docker.internal'
+    assert local.uses_host_gateway is True
+
+    intranet = resolve_pier_api_endpoint('http://10.1.2.3:8000/v1')
+    assert intranet is not None
+    assert intranet.base_url == 'http://10.1.2.3:8000/v1'
+    assert intranet.allowlist_host == '10.1.2.3'
+    assert intranet.uses_host_gateway is False
+
+    gateway = resolve_pier_api_endpoint('https://api.vectron.meta-stone.com/v1/chat/completions')
+    assert gateway is not None
+    assert gateway.base_url == 'https://api.vectron.meta-stone.com/v1'
+    assert gateway.allowlist_host == 'api.vectron.meta-stone.com'
+    assert gateway.uses_host_gateway is False
+
+
+def test_resolve_pier_model_name_openai_compat_and_native() -> None:
+    assert resolve_pier_model_name('DeepSeek/DeepSeek-V4-Flash-0731') == (
+        'openai/DeepSeek/DeepSeek-V4-Flash-0731'
+    )
+    assert resolve_pier_model_name('Qwen3-VL-8B-Instruct') == 'openai/Qwen3-VL-8B-Instruct'
+    assert resolve_pier_model_name('openai/gpt-5.5') == 'openai/gpt-5.5'
+    assert resolve_pier_model_name(
+        'anthropic/claude-opus-4-8',
+        route='native',
+    ) == 'anthropic/claude-opus-4-8'
+
+
+def test_pier_model_name_uses_openai_compat_by_default(tmp_path: Path) -> None:
+    class SlashModel:
+        name = 'DeepSeek/DeepSeek-V4-Flash-0731'
+
+    adapter = make_adapter(tmp_path)
+    assert adapter._pier_model_name(SlashModel()) == 'openai/DeepSeek/DeepSeek-V4-Flash-0731'
+
+    native = make_adapter(tmp_path, pier_route='native')
+    assert native._pier_model_name(SlashModel()) == 'DeepSeek/DeepSeek-V4-Flash-0731'
+
+
+def test_pier_allowed_hosts_come_from_api_url(tmp_path: Path) -> None:
+    cfg = TaskConfig(
+        datasets=['deep_swe'],
+        api_url='https://api.vectron.meta-stone.com/v1/chat/completions',
+        api_key='sk-test',
+        work_dir=str(tmp_path / 'outputs'),
+        dataset_args={'deep_swe': {'extra_params': {'extra_allowed_hosts': ['10.9.9.9']}}},
+    )
+    adapter = get_benchmark('deep_swe', cfg)
+    assert isinstance(adapter, DeepSWEAdapter)
+    assert adapter._pier_allowed_hosts() == ['api.vectron.meta-stone.com', '10.9.9.9']
+
+
 def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) -> None:
     captured: Dict[str, Any] = {}
     result = {
@@ -313,6 +374,7 @@ def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) -
     task_path = tmp_path / 'tasks' / 'task-a'
     task_path.mkdir(parents=True)
     adapter = make_adapter(tmp_path, pier_agent_kwargs={'model_class': 'litellm'})
+    adapter._task_config.api_url = 'http://10.1.2.3:8000/v1'
     sample = Sample(input='', metadata={'task_id': 'task-a', 'task_path': str(task_path)})
 
     result_dict = adapter._run_pier_job(MockModel(), sample)
@@ -321,7 +383,10 @@ def test_run_pier_job_uses_adhoc_task_source(monkeypatch: Any, tmp_path: Path) -
     assert captured['config'].tasks[0].path == task_path
     assert getattr(captured['config'].tasks[0], 'source', None) is None
     assert captured['config'].agents[0].kwargs == {'model_class': 'litellm'}
+    assert captured['config'].agents[0].model_name == 'openai/mock-model'
+    assert captured['config'].agents[0].extra_allowed_hosts == ['10.1.2.3']
     assert captured['config'].environment.type == 'docker'
+    assert getattr(captured['config'].environment, 'extra_docker_compose', None) in (None, [])
 
 
 @pytest.mark.skipif(

evalscope/tests/benchmark/test_terminal_bench.py

@@ -8,6 +8,7 @@ import pytest
 from evalscope.api.messages.perf_metrics import PerformanceMetrics
 from evalscope.api.metric import Score
 from evalscope.benchmarks.terminal_bench.terminal_bench_adapter import _phase_timeout_options, _TerminalBenchBase
+from evalscope.benchmarks.terminal_bench.utils import HARBOR_DEFAULT_CONTEXT_LIMIT, resolve_terminus_context_limit
 
 TRIAL_URI = 'file:///tmp/terminal-bench-trial'
 
@@ -145,3 +146,36 @@ def test_terminal_bench_trace_preserves_request_perf_metrics(tmp_path) -> None:
 
     assert messages[0].perf_metrics.latency == 1.0
     assert messages[0].perf_metrics.input_tokens == 3
+
+
+@pytest.mark.parametrize(
+    ('kwargs', 'expected'),
+    [
+        ({'context_limit': 65536, 'max_tokens': 8192, 'truncation_tokens': 131072}, 65536),
+        ({'max_tokens': 8192, 'truncation_tokens': 131072}, 8192),
+        ({'truncation_tokens': 65536}, 65536),
+        ({}, HARBOR_DEFAULT_CONTEXT_LIMIT),
+        ({'context_limit': 0, 'max_tokens': 8192, 'truncation_tokens': 65536}, 8192),
+        ({'context_limit': '', 'max_tokens': None, 'truncation_tokens': 65536}, 65536),
+    ],
+)
+def test_terminus_context_limit_priority(kwargs, expected) -> None:
+    assert resolve_terminus_context_limit(**kwargs) == expected
+
+
+def test_adapter_resolves_context_limit_from_extra_params() -> None:
+    adapter = object.__new__(_TerminalBenchBase)
+    adapter.context_limit = 65536
+    adapter.extra_params = {'truncation_tokens': 131072}
+    model = SimpleNamespace(config=SimpleNamespace(max_tokens=8192))
+
+    assert adapter._resolve_context_limit(model) == 65536
+
+    adapter.context_limit = None
+    assert adapter._resolve_context_limit(model) == 8192
+
+    model.config.max_tokens = None
+    assert adapter._resolve_context_limit(model) == 131072
+
+    adapter.extra_params = {}
+    assert adapter._resolve_context_limit(model) == HARBOR_DEFAULT_CONTEXT_LIMIT