chore: upgrade vendored evalscope to upstream v1.9.1 and reapply local patches

- Upgrade evalscope/evalscope from dev snapshot to upstream v1.9.1
- New benchmarks available: deep_swe, skillsbench, toolathlon,
  terminal_bench_v2_1, swe_bench_pro, browsecomp, gdpval, mcp_atlas, etc.
- Reapply local patches:
  - api/model/generate_config.py: add max_completion_tokens
  - api/model/model.py: treat EMPTY api_key as unset
  - models/utils/openai.py: pass max_completion_tokens; handle choice.index=None
  - benchmarks/swe_bench/utils.py: guard None instance_id/client
  - api/evaluator/cache.py: remove model_name from cache/report paths

evalscope/evalscope/agent/environments/enclave.py

@@ -19,9 +19,11 @@ shared ``atexit`` hook.
 
 from __future__ import annotations
 
+import re
 import shlex
 import time
-from typing import Any, Dict, List, Optional, Union
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Union
 
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.agent.types import ExecResult
@@ -42,6 +44,64 @@ logger = get_logger()
 _DEFAULT_DOCKER_IMAGE = 'python:3.11-slim'
 _DEFAULT_WORKDIR = '/workspace'
 _DEFAULT_TOOLS: List[str] = ['shell_executor', 'python_executor']
+_DEFAULT_INTERPRETER: List[str] = ['bash', '-c']
+_ENV_KEY_PATTERN = r'[A-Za-z_][A-Za-z0-9_]*'
+
+
+def _unwrap_bash_c(cmd: Any) -> Optional[str]:
+    """Return the payload for common bash-tool wrappers."""
+    if not isinstance(cmd, (list, tuple)) or len(cmd) != 3:
+        return None
+    executable, flag, command = cmd
+    if executable not in {'bash', '/bin/bash'} or flag != '-c' or not isinstance(command, str):
+        return None
+    return command
+
+
+def _interpreter_is_bash(interpreter: Sequence[str]) -> bool:
+    executable = interpreter[0]
+    return executable == 'bash' or executable.endswith('/bash')
+
+
+def _render_env_exports(env: Dict[str, str]) -> str:
+    exports = []
+    for raw_key, raw_value in env.items():
+        key = str(raw_key)
+        if not re.fullmatch(_ENV_KEY_PATTERN, key):
+            raise ValueError(f'Invalid environment variable name: {key!r}')
+        exports.append(f'export {key}={shlex.quote(str(raw_value))};')
+    return ' '.join(exports)
+
+
+def _render_command(
+    cmd: List[str],
+    *,
+    interpreter: Sequence[str],
+    cwd: Optional[str],
+    env: Optional[Dict[str, str]],
+) -> str:
+    unwrapped_command = _unwrap_bash_c(cmd) if _interpreter_is_bash(interpreter) else None
+    if unwrapped_command is not None:
+        command = unwrapped_command
+    elif isinstance(cmd, list):
+        command = ' '.join(shlex.quote(c) for c in cmd)
+    else:
+        command = str(cmd)
+    if cwd:
+        command = f'cd {shlex.quote(cwd)} && {command}'
+    if env:
+        prefix = _render_env_exports(env)
+        command = f'{prefix} {command}' if prefix else command
+    return command
+
+
+def _returncode_from_status(status: Any, stderr: str, timed_out: bool, success_status: Any) -> int:
+    if status == success_status:
+        return 0
+    if timed_out:
+        return -1
+    match = re.search(r'exit code (\d+)', stderr)
+    return int(match.group(1)) if match else 1
 
 
 @register_environment(['enclave', 'docker', 'volcengine'])
@@ -63,7 +123,12 @@ class EnclaveAgentEnvironment(AgentEnvironment):
         (e.g. ``base_url`` / ``region`` / credentials).
     timeout:
         Default command timeout (seconds) used when :meth:`exec` is called
-        without an explicit timeout.
+        without an explicit timeout. ``None`` uses the environment default.
+    interpreter:
+        Command interpreter argv prefix. The rendered command string is
+        appended as the final argument. Defaults to ``['bash', '-c']`` for
+        backward compatibility; benchmark adapters may use ``['bash', '-lc']``
+        when login-shell initialization is required.
     """
 
     name: str = 'enclave'
@@ -74,7 +139,8 @@ class EnclaveAgentEnvironment(AgentEnvironment):
         engine: Union[str, SandboxEngine] = SandboxEngine.DOCKER,
         sandbox_config: Optional[Dict[str, Any]] = None,
         manager_config: Optional[Dict[str, Any]] = None,
-        timeout: float = 60.0,
+        timeout: Optional[float] = None,
+        interpreter: Optional[Sequence[str]] = None,
         **_: Any,
     ) -> None:
         # ms_enclave is mandatory for this environment. Fail fast at construction
@@ -83,7 +149,18 @@ class EnclaveAgentEnvironment(AgentEnvironment):
         check_import('ms_enclave', extra='sandbox', raise_error=True, feature_name='EnclaveAgentEnvironment')
 
         self._engine: SandboxEngine = resolve_engine(engine)
-        self._timeout = float(timeout)
+        self._timeout = 60.0 if timeout is None else float(timeout)
+        if isinstance(interpreter, str):
+            raise TypeError(
+                'EnclaveAgentEnvironment.interpreter must be a non-empty sequence of non-empty strings, '
+                'not a single string.'
+            )
+        self._interpreter: List[str] = list(_DEFAULT_INTERPRETER if interpreter is None else interpreter)
+        invalid_interpreter = not self._interpreter or any(
+            not isinstance(part, str) or not part for part in self._interpreter
+        )
+        if invalid_interpreter:
+            raise ValueError('EnclaveAgentEnvironment.interpreter must be a non-empty sequence of non-empty strings.')
         self._manager_config: Dict[str, Any] = dict(manager_config or {})
         self._sandbox_config_dict: Dict[str, Any] = self._apply_engine_defaults(dict(sandbox_config or {}))
         self._handle: Optional[SandboxHandle] = None
@@ -156,24 +233,12 @@ class EnclaveAgentEnvironment(AgentEnvironment):
         env: Optional[Dict[str, str]] = None,
     ) -> ExecResult:
         handle = await self._ensure_sandbox()
-
-        if isinstance(cmd, list):
-            command = ' '.join(shlex.quote(c) for c in cmd)
-        else:
-            command = str(cmd)
-        if env:
-            # Prepend ``KEY=VAL`` pairs so the agent CLI inherits them.
-            # Using ``env -- ... <cmd>`` would be cleaner but is not always
-            # available inside minimal images; inline assignment works in any
-            # POSIX shell.
-            prefix = ' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items())
-            command = f'{prefix} {command}' if prefix else command
-        if cwd:
-            command = f'cd {shlex.quote(cwd)} && {command}'
+        command = _render_command(cmd, interpreter=self._interpreter, cwd=cwd, env=env)
 
         # ms_enclave's shell_executor splits a bare string with no shell
-        # wrapping; wrap as ``bash -c`` so cd/&&/env-prefix/quoting survive.
-        shell_argv = ['bash', '-c', command]
+        # wrapping; use an explicit interpreter so cd/&&/env-prefix/quoting
+        # survive, while allowing benchmarks to request login-shell setup.
+        shell_argv = [*self._interpreter, command]
 
         timeout_s = float(timeout or self._timeout)
 
@@ -195,15 +260,7 @@ class EnclaveAgentEnvironment(AgentEnvironment):
         stdout = str(result.output or '')
         stderr = str(result.error or '')
         timed_out = result.status == ExecutionStatus.TIMEOUT
-
-        if result.status == ExecutionStatus.SUCCESS:
-            returncode = 0
-        elif timed_out:
-            returncode = -1
-        else:
-            import re as _re
-            _m = _re.search(r'exit code (\d+)', stderr)
-            returncode = int(_m.group(1)) if _m else 1
+        returncode = _returncode_from_status(result.status, stderr, timed_out, ExecutionStatus.SUCCESS)
 
         # Prefer upstream-reported time when it's a positive number;
         # otherwise fall back to our locally measured wall-clock.
@@ -218,6 +275,13 @@ class EnclaveAgentEnvironment(AgentEnvironment):
             duration=duration,
         )
 
+    async def put_dir(self, source_dir: str | Path, target_dir: str) -> None:
+        """Copy a host directory into the sandbox."""
+        handle = await self._ensure_sandbox()
+        ok = await handle.put_dir(source_dir, target_dir)
+        if not ok:
+            raise RuntimeError(f'EnclaveAgentEnvironment.put_dir failed to copy {source_dir} into {target_dir}')
+
     async def close(self) -> None:
         """Release the per-sample sandbox (idempotent)."""
         if self._handle is not None:

evalscope/evalscope/agent/environments/local.py

@@ -6,6 +6,9 @@ No container isolation - suitable only for development and CI tests.
 
 import asyncio
 import os
+import shutil
+import tempfile
+from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from evalscope.api.agent import AgentEnvironment
@@ -86,5 +89,38 @@ class LocalAgentEnvironment(AgentEnvironment):
     async def close(self) -> None:
         """No external resources to release."""
 
+    async def put_dir(self, source_dir: str | Path, target_dir: str) -> None:
+        """Copy a host directory into a local target directory."""
+        source = Path(source_dir).expanduser()
+        if not source.is_dir():
+            raise FileNotFoundError(f'put_dir source is not a directory: {source}')
+        target = Path(target_dir).expanduser()
+        target.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(source, target, dirs_exist_ok=True)
 
-__all__ = ['LocalAgentEnvironment']
+
+class TemporaryLocalAgentEnvironment(LocalAgentEnvironment):
+    """Local environment backed by a temporary working directory."""
+
+    def __init__(
+        self,
+        sample_id: Any = None,
+        *,
+        prefix: str = 'evalscope-local-',
+        env_vars: Optional[Dict[str, str]] = None,
+    ) -> None:
+        raw_sample_id = 'sample' if sample_id is None else str(sample_id)
+        safe_id = ''.join(char if char.isalnum() else '-' for char in raw_sample_id)[:64]
+        self._temporary_directory = tempfile.TemporaryDirectory(prefix=f'{prefix}{safe_id}-')
+        super().__init__(working_dir=self._temporary_directory.name, env_vars=env_vars)
+
+    @property
+    def working_dir(self) -> Path:
+        return Path(self._temporary_directory.name)
+
+    async def close(self) -> None:
+        await super().close()
+        self._temporary_directory.cleanup()
+
+
+__all__ = ['LocalAgentEnvironment', 'TemporaryLocalAgentEnvironment']

evalscope/evalscope/agent/external/adapter.py

@@ -15,6 +15,12 @@ import platform
 import time
 from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional
 
+from evalscope.agent.skills import (
+    DEFAULT_SKILLS_INSTALL_DIR,
+    ResolvedSkills,
+    format_skills_prompt,
+    resolve_agent_skills,
+)
 from evalscope.api.agent import AgentEnvironment, AgentTrace
 from evalscope.api.evaluator import InferenceResult
 from evalscope.api.messages import ChatMessageAssistant, ChatMessageSystem, ChatMessageUser
@@ -47,6 +53,7 @@ def run_external_agent(
     environment_override: Optional[AgentEnvironment] = None,
     instruction_override: Optional[str] = None,
     post_run_hook: Optional[PostRunHook] = None,
+    close_environment: bool = True,
 ) -> InferenceResult:
     """Synchronously drive one sample through an external agent runner.
 
@@ -73,21 +80,40 @@ def run_external_agent(
         value replaces ``run_result.output`` as the InferenceResult text
         — the typical use is ``extract_patch(env, cwd)`` for SWE-bench
         adapters that recover a ``git diff`` from the working tree.
+    close_environment:
+        Whether this function owns and closes the environment. Set to
+        ``False`` when passing a caller-owned ``environment_override`` that
+        must remain open after the external runner finishes.
 
     Uses :class:`AsyncioLoopRunner` to submit the coroutine to the calling
     thread's long-lived background loop.  That loop is reused across
     samples so the :class:`ModelProxyServer` singleton (which binds to it)
     only spins up once per worker thread instead of once per sample.
     """
+    if environment_override is None and not close_environment:
+        raise ValueError('close_environment=False requires environment_override')
+
     instruction = instruction_override if instruction_override is not None else _instruction_from_sample(sample)
+    skills = resolve_agent_skills(
+        sample_metadata=sample.metadata,
+        config_skills_dir=config.skills_dir,
+        prompt_base_dir=DEFAULT_SKILLS_INSTALL_DIR,
+        install_paths=[DEFAULT_SKILLS_INSTALL_DIR],
+    )
+    if config.skill_prompt_nudge and skills.enabled:
+        nudge = format_skills_prompt(skills.skills)
+        if nudge:
+            instruction = f'{nudge}\n\n{instruction}'
     return AsyncioLoopRunner.run(
         _run_async(
             config=config,
             model=model,
             sample=sample,
             instruction=instruction,
+            skills=skills,
             environment_override=environment_override,
             post_run_hook=post_run_hook,
+            close_environment=close_environment,
         )
     )
 
@@ -97,8 +123,10 @@ async def _run_async(
     model: Model,
     sample: 'Sample',
     instruction: str,
+    skills: ResolvedSkills,
     environment_override: Optional[AgentEnvironment],
     post_run_hook: Optional[PostRunHook],
+    close_environment: bool,
 ) -> InferenceResult:
     runner_cls = get_runner(config.framework)
     runner_kwargs = dict(config.kwargs)
@@ -131,9 +159,13 @@ async def _run_async(
         task = ExternalAgentTask(
             instruction=instruction,
             timeout=config.timeout,
-            metadata={'sample_id': getattr(sample, 'id', None)},
+            metadata={
+                'sample_id': getattr(sample, 'id', None),
+                'agent_skills': skills.model_dump(),
+            },
         )
-        async with env:
+
+        async def run_in_environment() -> str:
             session.recorder.record_run_start(
                 framework=config.framework,
                 cmd_summary=runner_cls.__name__,
@@ -169,9 +201,14 @@ async def _run_async(
             # still query the sandbox (e.g. extract a ``git diff``) before
             # the per-sample environment is closed.
             if post_run_hook is not None:
-                final_text = await post_run_hook(env, result, sample)
-            else:
-                final_text = result.output
+                return await post_run_hook(env, result, sample)
+            return result.output
+
+        if close_environment:
+            async with env:
+                final_text = await run_in_environment()
+        else:
+            final_text = await run_in_environment()
 
         trace: AgentTrace = session.recorder.snapshot()
         # Prefer the env's own ``name`` (set on the AgentEnvironment subclass)

evalscope/evalscope/agent/external/bridge/server.py

@@ -914,6 +914,10 @@ def _build_generate_config(body: Dict[str, Any]) -> GenerateConfig:
         kwargs['top_p'] = body['top_p']
     if 'stop_sequences' in body and body['stop_sequences']:
         kwargs['stop_seqs'] = list(body['stop_sequences'])
+    cache_control = body.get('cache_control')
+    if isinstance(cache_control, dict):
+        kwargs['anthropic_cache_control'] = cache_control
+        kwargs['anthropic_cache_strategy'] = 'recent_messages'
     return GenerateConfig(**kwargs)
 
 

evalscope/evalscope/agent/external/bridge/sse_anthropic.py

@@ -25,7 +25,7 @@ from typing import Any, AsyncIterator, Dict, Optional
 
 from evalscope.api.model import ModelOutput
 from ._sse_common import PING_INTERVAL_S, TEXT_CHUNK, TOOL_INPUT_CHUNK, iter_chunks
-from .translate_anthropic import map_stop_reason_to_anthropic, unpack_tool_call
+from .translate_anthropic import anthropic_usage_payload, map_stop_reason_to_anthropic, unpack_tool_call
 
 
 def _sse(event_type: str, data: Dict[str, Any]) -> bytes:
@@ -107,17 +107,13 @@ async def stream_anthropic_response(
 
     # 4. message_delta — stop reason + cumulative usage.
     stop_reason = map_stop_reason_to_anthropic(output.choices[0].stop_reason if output.choices else 'stop')
-    usage = output.usage
     delta_payload: Dict[str, Any] = {
         'type': 'message_delta',
         'delta': {
             'stop_reason': stop_reason,
             'stop_sequence': None
         },
-        'usage': {
-            'input_tokens': usage.input_tokens if usage else 0,
-            'output_tokens': usage.output_tokens if usage else 0,
-        },
+        'usage': anthropic_usage_payload(output.usage),
     }
     yield _sse('message_delta', delta_payload)
 

evalscope/evalscope/agent/external/bridge/translate_anthropic.py

@@ -1,7 +1,8 @@
 """Anthropic Messages API ⇄ EvalScope native type translation.
 
-P0 scope: text + ``tool_use`` + ``tool_result`` blocks, no streaming, no
-``cache_control``, no extended-thinking blocks.  See
+P0 scope: text + ``tool_use`` + ``tool_result`` blocks, no extended-thinking
+blocks.  Anthropic ``cache_control`` markers are preserved through EvalScope
+provider-specific ``internal`` / ``options`` fields.  See
 ``.qoder/plans/agent_bridge_design.md`` §7.4 for known-lossy cases.
 """
 
@@ -14,6 +15,7 @@ from evalscope.api.messages import (
     ChatMessageSystem,
     ChatMessageTool,
     ChatMessageUser,
+    ContentText,
 )
 from evalscope.api.model import ModelOutput
 from evalscope.api.tool import ToolCall, ToolCallError, ToolFunction, ToolInfo, ToolParams
@@ -47,9 +49,10 @@ def anthropic_request_to_messages(body: Dict[str, Any]) -> List[ChatMessage]:
     if isinstance(system, str) and system:
         messages.append(ChatMessageSystem(content=system))
     elif isinstance(system, list):
-        text = ''.join(b.get('text', '') for b in system if isinstance(b, dict) and b.get('type') == 'text')
-        if text:
-            messages.append(ChatMessageSystem(content=text))
+        content = [_content_text_from_block(b) for b in system if isinstance(b, dict) and b.get('type') == 'text']
+        content = [c for c in content if c.text]
+        if content:
+            messages.append(ChatMessageSystem(content=content))
 
     for entry in body.get('messages') or []:
         if not isinstance(entry, dict):
@@ -68,21 +71,21 @@ def _user_blocks_to_messages(content: Any) -> List[ChatMessage]:
         return [ChatMessageUser(content=content)]
     if not isinstance(content, list):
         return []
-    user_text_parts: List[str] = []
+    user_content: List[ContentText] = []
     tool_msgs: List[ChatMessage] = []
     for block in content:
         if not isinstance(block, dict):
             continue
         btype = block.get('type')
         if btype == 'text':
-            user_text_parts.append(block.get('text', ''))
+            user_content.append(_content_text_from_block(block))
         elif btype == 'tool_result':
             tool_msgs.append(_tool_result_to_message(block))
     # Tool results precede any new user text so the model sees the
     # observation first, then the new prompt (matches OpenAI ordering).
     out: List[ChatMessage] = list(tool_msgs)
-    if user_text_parts:
-        out.append(ChatMessageUser(content='\n'.join(p for p in user_text_parts if p)))
+    if user_content:
+        out.append(ChatMessageUser(content=_content_or_text(user_content)))
     return out
 
 
@@ -98,13 +101,14 @@ def _tool_result_to_message(block: Dict[str, Any]) -> ChatMessageTool:
         content=text,
         tool_call_id=block.get('tool_use_id'),
         error=error,
+        internal=_anthropic_internal_from_block(block),
     )
 
 
 def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant:
     if isinstance(content, str):
         return ChatMessageAssistant(content=content)
-    text_parts: List[str] = []
+    text_content: List[ContentText] = []
     tool_calls: List[ToolCall] = []
     if isinstance(content, list):
         for block in content:
@@ -112,7 +116,7 @@ def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant:
                 continue
             btype = block.get('type')
             if btype == 'text':
-                text_parts.append(block.get('text', ''))
+                text_content.append(_content_text_from_block(block))
             elif btype == 'tool_use':
                 tool_calls.append(
                     ToolCall(
@@ -121,15 +125,43 @@ def _assistant_blocks_to_message(content: Any) -> ChatMessageAssistant:
                             name=block.get('name', ''),
                             arguments=block.get('input') or {},
                         ),
+                        internal=_anthropic_internal_from_block(block),
                         type='function',
                     )
                 )
     return ChatMessageAssistant(
-        content='\n'.join(p for p in text_parts if p),
+        content=_content_or_text(text_content),
         tool_calls=tool_calls or None,
     )
 
 
+def _content_text_from_block(block: Dict[str, Any]) -> ContentText:
+    return ContentText(
+        text=block.get('text', ''),
+        internal=_anthropic_internal_from_block(block),
+    )
+
+
+def _content_or_text(content: List[ContentText]) -> Any:
+    if any(_has_anthropic_cache_control(c.internal) for c in content):
+        return content
+    return '\n'.join(c.text for c in content if c.text)
+
+
+def _anthropic_internal_from_block(block: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    cache_control = block.get('cache_control')
+    if isinstance(cache_control, dict):
+        return {'anthropic': {'cache_control': cache_control}}
+    return None
+
+
+def _has_anthropic_cache_control(internal: Any) -> bool:
+    return (
+        isinstance(internal, dict) and isinstance(internal.get('anthropic'), dict)
+        and isinstance(internal['anthropic'].get('cache_control'), dict)
+    )
+
+
 def anthropic_tools_to_tool_infos(tools: Sequence[Dict[str, Any]]) -> List[ToolInfo]:
     """Translate Anthropic tool specs to ``ToolInfo``.  Best-effort: any
     unparsable parameter schema falls back to an empty ``ToolParams``."""
@@ -142,11 +174,14 @@ def anthropic_tools_to_tool_infos(tools: Sequence[Dict[str, Any]]) -> List[ToolI
             continue
         schema = spec.get('input_schema') or {}
         params = ToolParams() if not isinstance(schema, dict) else _safe_tool_params(schema)
-        out.append(ToolInfo(
-            name=name,
-            description=spec.get('description', '') or '',
-            parameters=params,
-        ))
+        out.append(
+            ToolInfo(
+                name=name,
+                description=spec.get('description', '') or '',
+                parameters=params,
+                options=_anthropic_internal_from_block(spec),
+            )
+        )
     return out
 
 
@@ -185,7 +220,7 @@ def model_output_to_anthropic_response(
         blocks.append({'type': 'text', 'text': ''})
 
     stop_reason = map_stop_reason_to_anthropic(output.choices[0].stop_reason if output.choices else 'stop')
-    usage = output.usage
+    usage = anthropic_usage_payload(output.usage)
     return {
         'id': output.id or f'msg_{uuid.uuid4().hex[:24]}',
         'type': 'message',
@@ -194,10 +229,7 @@ def model_output_to_anthropic_response(
         'model': request_model or output.model or '',
         'stop_reason': stop_reason,
         'stop_sequence': None,
-        'usage': {
-            'input_tokens': usage.input_tokens if usage else 0,
-            'output_tokens': usage.output_tokens if usage else 0,
-        },
+        'usage': usage,
     }
 
 
@@ -218,3 +250,15 @@ def map_stop_reason_to_anthropic(reason: str) -> str:
     table lives in exactly one place.
     """
     return _STOP_REASON_MAP.get(reason, 'end_turn')
+
+
+def anthropic_usage_payload(usage: Any) -> Dict[str, Any]:
+    payload: Dict[str, Any] = {
+        'input_tokens': usage.input_tokens if usage else 0,
+        'output_tokens': usage.output_tokens if usage else 0,
+    }
+    if usage and usage.input_tokens_cache_write is not None:
+        payload['cache_creation_input_tokens'] = usage.input_tokens_cache_write
+    if usage and usage.input_tokens_cache_read is not None:
+        payload['cache_read_input_tokens'] = usage.input_tokens_cache_read
+    return payload

evalscope/evalscope/agent/external/runners/claude_code.py

@@ -19,8 +19,8 @@ from typing import Any, Dict, List, Optional
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.registry import register_runner
 from evalscope.utils.logger import get_logger
-from ._node_install import ensure_node_via_apt
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import ensure_node_via_apt, install_task_skills
 
 logger = get_logger()
 
@@ -181,6 +181,13 @@ class ClaudeCodeRunner(AgentRunner):
             env_vars['HOME'] = home_dir
 
         try:
+            await install_task_skills(
+                env,
+                task,
+                home_dir=home_dir,
+                native_install_paths=['$HOME/.claude/skills'],
+                runner_name='ClaudeCodeRunner',
+            )
             # Pass the prompt as the trailing positional argument (matches
             # claude-code's documented invocation pattern).  Avoid variadic
             # flags like ``--allowedTools <tools...>`` before the positional

evalscope/evalscope/agent/external/runners/codex.py

@@ -22,8 +22,8 @@ from typing import Any, Dict, List, Optional
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.registry import register_runner
 from evalscope.utils.logger import get_logger
-from ._node_install import ensure_node_via_apt
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import ensure_node_via_apt, install_task_skills
 
 logger = get_logger()
 
@@ -170,6 +170,13 @@ class CodexRunner(AgentRunner):
         home_dir = self._resolve_home()
         if home_dir is not None:
             env_vars['HOME'] = home_dir
+        await install_task_skills(
+            env,
+            task,
+            home_dir=home_dir,
+            native_install_paths=['$HOME/.agents/skills'],
+            runner_name='CodexRunner',
+        )
 
         # Build -c overrides. Order: builtin (provider config) → user extras.
         # codex parses these as TOML literals, so string values need shell-

evalscope/evalscope/agent/external/runners/gemini_cli.py

@@ -23,8 +23,8 @@ from typing import Any, Dict, List, Optional
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.registry import register_runner
 from evalscope.utils.logger import get_logger
-from ._node_install import ensure_node_via_apt
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import ensure_node_via_apt, install_task_skills
 
 logger = get_logger()
 
@@ -146,6 +146,13 @@ class GeminiCliRunner(AgentRunner):
         home_dir = self._resolve_home()
         if home_dir is not None:
             env_vars['HOME'] = home_dir
+        await install_task_skills(
+            env,
+            task,
+            home_dir=home_dir,
+            native_install_paths=['$HOME/.gemini/skills'],
+            runner_name='GeminiCliRunner',
+        )
 
         # Build the command.
         # gemini -p "prompt" executes non-interactively.

evalscope/evalscope/agent/external/runners/hermes.py

@@ -28,6 +28,7 @@ from evalscope.api.agent import AgentEnvironment
 from evalscope.api.registry import register_runner
 from evalscope.utils.logger import get_logger
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import install_task_skills
 
 logger = get_logger()
 
@@ -175,6 +176,13 @@ class HermesRunner(AgentRunner):
         }
         if home_dir is not None:
             env_vars['HERMES_HOME'] = home_dir
+        await install_task_skills(
+            env,
+            task,
+            home_dir=None,
+            native_install_paths=[],
+            runner_name='HermesRunner',
+        )
 
         # Write a config.yaml that points Hermes at the bridge endpoint.
         # When base_url is set, Hermes ignores the provider and calls the

evalscope/evalscope/agent/external/runners/install_helper.py

@@ -1,18 +1,13 @@
-"""Shared Node.js probe and nodesource apt installer for Node-based runners.
+"""Installation helpers shared by external runners."""
 
-Four runners (``claude-code``, ``codex``, ``opencode``, ``gemini-cli``) all
-need the same two-step sequence when Node.js is absent: install apt
-prerequisites (curl, ca-certificates, gnupg) via apt-get, then pull the
-nodesource setup script and install ``nodejs``.  This module extracts that
-common path so the logic lives in one place.
-"""
-
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Optional
 
+from evalscope.agent.skills import install_agent_skills, skills_from_sample_metadata
 from evalscope.utils.logger import get_logger
 
 if TYPE_CHECKING:
     from evalscope.api.agent import AgentEnvironment
+    from .base import ExternalAgentTask
 
 logger = get_logger()
 
@@ -30,18 +25,7 @@ async def ensure_node_via_apt(
     timeout_s: float,
     runner_name: str,
 ) -> None:
-    """Ensure Node.js and npm are available, installing via nodesource if needed.
-
-    No-ops when ``node`` and ``npm`` are already on PATH.  Raises
-    ``RuntimeError`` when any install step fails (apt prereqs or nodesource).
-
-    Args:
-        env: The agent execution environment.
-        node_setup_url: URL of the nodesource distribution setup script
-            (e.g. ``https://deb.nodesource.com/setup_22.x``).
-        timeout_s: Wall-clock budget (seconds) for each sub-command.
-        runner_name: Runner class name used in log and error messages.
-    """
+    """Ensure Node.js and npm are available, installing via nodesource if needed."""
     if await node_present(env):
         return
     logger.info(
@@ -82,4 +66,38 @@ async def ensure_node_via_apt(
         )
 
 
-__all__ = ['ensure_node_via_apt', 'node_present']
+async def install_task_skills(
+    env: 'AgentEnvironment',
+    task: 'ExternalAgentTask',
+    *,
+    home_dir: Optional[str],
+    native_install_paths: Optional[List[str]] = None,
+    runner_name: str,
+) -> None:
+    """Copy task skills into paths visible to the wrapped agent CLI."""
+    skills = skills_from_sample_metadata(task.metadata)
+    if not skills.enabled or not skills.sandbox_dir:
+        return
+
+    install_paths = _resolve_install_paths(
+        list(skills.install_paths or []) + list(native_install_paths or []),
+        home_dir=home_dir,
+    )
+    await install_agent_skills(env, skills, install_paths=install_paths, runner_name=runner_name)
+
+
+def _resolve_install_paths(install_paths: List[str], *, home_dir: Optional[str]) -> List[str]:
+    resolved: List[str] = []
+    seen = set()
+    for path in install_paths:
+        if not path:
+            continue
+        resolved_path = path.replace('$HOME', home_dir) if home_dir else path
+        if resolved_path in seen:
+            continue
+        seen.add(resolved_path)
+        resolved.append(resolved_path)
+    return resolved
+
+
+__all__ = ['ensure_node_via_apt', 'install_task_skills', 'node_present']

evalscope/evalscope/agent/external/runners/mock.py

@@ -14,6 +14,7 @@ from typing import Any, Dict
 
 from evalscope.api.registry import register_runner
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import install_task_skills
 
 # The body of the mock-agent CLI.  Lives as a string so it runs in any
 # Python interpreter available inside the sandbox (no module import).
@@ -97,6 +98,13 @@ class MockAgentRunner(AgentRunner):
             env_vars['ANTHROPIC_MODEL'] = self._model_name
         if task.timeout is not None:
             env_vars['MOCK_TIMEOUT'] = str(task.timeout)
+        await install_task_skills(
+            env,
+            task,
+            home_dir=None,
+            native_install_paths=[],
+            runner_name='MockAgentRunner',
+        )
 
         result = await env.exec(
             [sys.executable, '-c', _MOCK_AGENT_SCRIPT],

evalscope/evalscope/agent/external/runners/opencode.py

@@ -29,8 +29,8 @@ from typing import Any, Dict, List, Optional
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.registry import register_runner
 from evalscope.utils.logger import get_logger
-from ._node_install import ensure_node_via_apt
 from .base import AgentRunner, AgentRunResult, BridgeEndpoint, ExternalAgentTask, RunnerTimeoutError
+from .install_helper import ensure_node_via_apt, install_task_skills
 
 logger = get_logger()
 
@@ -164,6 +164,13 @@ class OpenCodeRunner(AgentRunner):
             env_vars['HOME'] = home_dir
 
         try:
+            await install_task_skills(
+                env,
+                task,
+                home_dir=home_dir,
+                native_install_paths=['$HOME/.config/opencode/skills', '$HOME/.opencode/skill'],
+                runner_name='OpenCodeRunner',
+            )
             # Write opencode.json config to register the model + baseURL.
             config = {
                 'provider': {

evalscope/evalscope/agent/runner.py

@@ -13,11 +13,19 @@ per-benchmark overrides.
 
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
 
+from evalscope.agent.skills import (
+    DEFAULT_SKILLS_INSTALL_DIR,
+    format_skills_prompt,
+    install_agent_skills,
+    resolve_agent_skills,
+)
+from evalscope.agent.tools.bash import apply_bash_command_timeout_defaults
 from evalscope.api.agent import AgentEnvironment, AgentLoopResult, run_agent_loop
 from evalscope.api.evaluator import InferenceResult
 from evalscope.api.messages import ChatMessageUser
 from evalscope.api.model import Model
 from evalscope.api.registry import get_environment, get_strategy, resolve_tool_infos, resolve_tools
+from evalscope.utils.function_utils import AsyncioLoopRunner
 from evalscope.utils.logger import get_logger
 
 if TYPE_CHECKING:
@@ -34,6 +42,7 @@ def run_native_agent(
     sample: 'Sample',
     build_sandbox_config: Callable[['Sample'], Optional[Dict[str, Any]]],
     extract_final_answer: Callable[[AgentLoopResult, Any], str],
+    environment_override: Optional[AgentEnvironment] = None,
 ) -> InferenceResult:
     """Drive a sample through the native AgentLoop and return its result.
 
@@ -62,27 +71,43 @@ def run_native_agent(
     # Resolve ToolInfo schemas from the registry so the model can see them.
     registered_tool_infos = resolve_tool_infos(cfg.tools)
 
-    # Determine environment class (if any) – instantiated below so its
-    # constructor sees the fully merged kwargs.
-    env_cls: Optional[type] = None
-    if cfg.environment is not None:
+    environment: Optional[AgentEnvironment] = environment_override
+    if environment is None and cfg.environment is not None:
         env_cls = get_environment(cfg.environment)
-
-    env_kwargs = _resolve_env_kwargs(
-        task_config=task_config,
-        sample=sample,
-        build_sandbox_config=build_sandbox_config,
-    )
-    environment: Optional[AgentEnvironment] = env_cls(**env_kwargs) if env_cls is not None else None
+        env_kwargs = _resolve_env_kwargs(
+            task_config=task_config,
+            sample=sample,
+            build_sandbox_config=build_sandbox_config,
+        )
+        environment = env_cls(**env_kwargs)
+    owns_environment = environment_override is None
 
     if isinstance(sample.input, list):
         initial_messages = list(sample.input)
     else:
         initial_messages = [ChatMessageUser(content=sample.input)]
+    skills = resolve_agent_skills(
+        sample_metadata=sample.metadata,
+        config_skills_dir=cfg.skills_dir,
+        prompt_base_dir=DEFAULT_SKILLS_INSTALL_DIR,
+        install_paths=[DEFAULT_SKILLS_INSTALL_DIR],
+    )
+    if cfg.skill_prompt_nudge and skills.enabled:
+        nudge = format_skills_prompt(skills.skills)
+        if nudge:
+            initial_messages.insert(0, ChatMessageUser(content=nudge))
+    if environment is not None:
+        try:
+            AsyncioLoopRunner.run(install_agent_skills(environment, skills, runner_name='NativeAgentRunner'))
+        except Exception:
+            if owns_environment:
+                AsyncioLoopRunner.run(environment.close())
+            raise
 
     # Merge sample-level tools with agent-config tools.
     sample_tools = list(sample.tools or [])
     all_tools = sample_tools + [t for t in registered_tool_infos if t not in sample_tools]
+    handlers, all_tools = apply_bash_command_timeout_defaults(handlers, all_tools, cfg.command_timeout)
 
     result: AgentLoopResult = run_agent_loop(
         model=model,
@@ -96,6 +121,7 @@ def run_native_agent(
         trace_strategy_name=cfg.strategy,
         trace_env_name=cfg.environment,
         mcp_configs=list(cfg.mcp_servers) or None,
+        close_environment=owns_environment,
     )
 
     final_text = extract_final_answer(result, strategy)
@@ -114,10 +140,11 @@ def _resolve_env_kwargs(
 
     Precedence (lowest -> highest):
       1. ``task_config.sandbox`` — engine / default_config / manager_config
-         carried alongside the pooled SandboxMixin so sandbox settings are
+         carried alongside the pooled CodeExecutionSandboxMixin so sandbox settings are
          defined **once** at the task level.
       2. ``build_sandbox_config(sample)`` — per-sample override hook.
-      3. ``agent_config.environment_extra`` — raw kwargs forwarded verbatim
+      3. ``agent_config.command_timeout`` — default timeout for command-style environments.
+      4. ``agent_config.environment_extra`` — raw kwargs forwarded verbatim
          to the environment constructor (last word for power users).
     """
     env_kwargs: Dict[str, Any] = {}
@@ -140,6 +167,9 @@ def _resolve_env_kwargs(
     if merged_sandbox_cfg:
         env_kwargs['sandbox_config'] = merged_sandbox_cfg
 
+    if task_config.agent_config.command_timeout is not None:
+        env_kwargs['timeout'] = task_config.agent_config.command_timeout
+
     # environment_extra wins over everything above.
     env_kwargs.update(task_config.agent_config.environment_extra)
     return env_kwargs

evalscope/evalscope/agent/skills.py

@@ -0,0 +1,234 @@
+"""Helpers for Agent Skills directory compatibility."""
+
+from __future__ import annotations
+
+import re
+import shlex
+import yaml
+from pathlib import Path
+from pydantic import BaseModel, Field
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List
+
+if TYPE_CHECKING:
+    from evalscope.api.agent import AgentEnvironment
+
+CONFIG_SKILL_SOURCE = 'config'
+TASK_BUNDLED_SKILL_SOURCE = 'task_bundled'
+DEFAULT_SKILLS_SANDBOX_DIR = '/tmp/evalscope-agent-skills'
+DEFAULT_SKILLS_INSTALL_DIR = '$HOME/.agents/skills'
+
+
+class SkillMetadata(BaseModel):
+    """Metadata discovered from a ``SKILL.md`` file."""
+
+    name: str
+    description: str = ''
+    path: str
+
+
+class ResolvedSkills(BaseModel):
+    """Resolved skills for one agent run."""
+
+    enabled: bool = False
+    source: str = 'none'
+    host_dir: str | None = None
+    sandbox_dir: str | None = None
+    prompt_base_dir: str | None = None
+    install_paths: List[str] = Field(default_factory=list)
+    skills: List[SkillMetadata] = Field(default_factory=list)
+    metadata_errors: List[str] = Field(default_factory=list)
+
+
+def discover_skills(skills_dir: str | Path, *, path_prefix: str | None = None) -> tuple[List[SkillMetadata], List[str]]:
+    """Discover immediate child skill directories containing ``SKILL.md``."""
+    base = Path(skills_dir)
+    skills: List[SkillMetadata] = []
+    errors: List[str] = []
+    if not base.is_dir():
+        return skills, [f'skills_dir is not a directory: {base}']
+
+    for skill_dir in sorted(path for path in base.iterdir() if path.is_dir()):
+        skill_file = skill_dir / 'SKILL.md'
+        if not skill_file.is_file():
+            continue
+        try:
+            content = skill_file.read_text(encoding='utf-8')
+        except OSError as exc:
+            errors.append(f'{skill_file}: {exc}')
+            continue
+        frontmatter = parse_frontmatter(content)
+        name = frontmatter.get('name') or skill_dir.name
+        description = frontmatter.get('description') or ''
+        if not frontmatter.get('name') or not frontmatter.get('description'):
+            errors.append(f'{skill_file}: missing name or description frontmatter')
+        display_base = (path_prefix.rstrip('/') if path_prefix else str(skill_dir.parent))
+        skills.append(
+            SkillMetadata(
+                name=name,
+                description=description,
+                path=f'{display_base}/{skill_dir.name}/SKILL.md',
+            )
+        )
+    return skills, errors
+
+
+def parse_frontmatter(content: str) -> Dict[str, str]:
+    """Parse YAML frontmatter key/value pairs from a skill file."""
+    match = re.match(r'^---\s*\n(.*?)\n---', content, re.DOTALL)
+    if not match:
+        return {}
+    parsed = yaml.safe_load(match.group(1)) or {}
+    if not isinstance(parsed, dict):
+        return {}
+    return {str(key): '' if value is None else str(value) for key, value in parsed.items()}
+
+
+def format_skills_prompt(skills: List[SkillMetadata]) -> str:
+    """Render a neutral prompt nudge for available skills."""
+    if not skills:
+        return ''
+    lines = [
+        'You have access to the following skills. Each skill is a directory containing a SKILL.md file with '
+        'instructions and resources. When a skill is relevant, read its SKILL.md before using it.',
+        '',
+    ]
+    for skill in skills:
+        description = f': {skill.description}' if skill.description else ''
+        lines.append(f'- {skill.name}{description} ({skill.path})')
+    return '\n'.join(lines)
+
+
+def skills_from_sample_metadata(metadata: Dict[str, Any]) -> ResolvedSkills:
+    """Build ``ResolvedSkills`` from sample metadata."""
+    raw = metadata.get('agent_skills') or {}
+    if isinstance(raw, ResolvedSkills):
+        return raw
+    if isinstance(raw, dict):
+        try:
+            return ResolvedSkills.model_validate(raw)
+        except Exception:
+            return ResolvedSkills()
+    return ResolvedSkills()
+
+
+def resolve_agent_skills(
+    *,
+    sample_metadata: Dict[str, Any],
+    config_skills_dir: str | None,
+    prompt_base_dir: str = DEFAULT_SKILLS_INSTALL_DIR,
+    install_paths: Iterable[str] = (DEFAULT_SKILLS_INSTALL_DIR, ),
+    sandbox_dir: str = DEFAULT_SKILLS_SANDBOX_DIR,
+) -> ResolvedSkills:
+    """Resolve sample-bundled skills first, then user-configured skills."""
+    sample_skills = skills_from_sample_metadata(sample_metadata)
+    if sample_skills.enabled:
+        return sample_skills
+    if not config_skills_dir:
+        return ResolvedSkills()
+
+    host_dir = Path(config_skills_dir).expanduser()
+    if not host_dir.is_dir():
+        raise FileNotFoundError(f'skills_dir is not a directory: {host_dir}')
+
+    skills, errors = discover_skills(host_dir, path_prefix=prompt_base_dir)
+    return ResolvedSkills(
+        enabled=bool(skills),
+        source=CONFIG_SKILL_SOURCE,
+        host_dir=str(host_dir),
+        sandbox_dir=sandbox_dir,
+        prompt_base_dir=prompt_base_dir,
+        install_paths=list(install_paths),
+        skills=skills,
+        metadata_errors=errors,
+    )
+
+
+async def install_agent_skills(
+    environment: 'AgentEnvironment',
+    skills: ResolvedSkills,
+    *,
+    install_paths: Iterable[str] | None = None,
+    runner_name: str,
+    timeout: float = 60,
+) -> None:
+    """Stage and install resolved skills into paths visible to an agent."""
+    if not skills.enabled:
+        return
+    await stage_agent_skills(environment, skills, runner_name=runner_name)
+
+    resolved_install_paths = _dedupe_paths(list(install_paths) if install_paths is not None else skills.install_paths)
+    command = install_skills_command(skills.sandbox_dir or '', resolved_install_paths)
+    if not command:
+        return
+    result = await environment.exec(['bash', '-lc', command], timeout=timeout)
+    if result.returncode != 0:
+        detail = ((result.stderr or result.stdout or '').strip() or f'rc={result.returncode}')[-1000:]
+        raise RuntimeError(f'{runner_name} failed to install skills: {detail}')
+
+
+async def stage_agent_skills(environment: 'AgentEnvironment', skills: ResolvedSkills, *, runner_name: str) -> None:
+    """Upload host-configured skills into the environment when needed."""
+    if not skills.enabled or skills.source != CONFIG_SKILL_SOURCE:
+        return
+    if not skills.host_dir or not skills.sandbox_dir:
+        raise RuntimeError(f'{runner_name} received config skills without host_dir and sandbox_dir')
+    try:
+        await environment.put_dir(skills.host_dir, skills.sandbox_dir)
+    except NotImplementedError as exc:
+        raise RuntimeError(f'{runner_name} requires environment.put_dir to install skills_dir') from exc
+    except Exception as exc:
+        raise RuntimeError(f'{runner_name} failed to stage skills: {exc}') from exc
+
+
+def install_skills_command(source_dir: str, install_paths: List[str]) -> str | None:
+    """Return a POSIX shell command copying skills into discovery paths."""
+    if not source_dir or not install_paths:
+        return None
+    commands = []
+    quoted_source = shlex.quote(source_dir.rstrip('/'))
+    for dest in install_paths:
+        quoted_dest = quote_path_with_home(dest.rstrip('/'))
+        commands.append(f'mkdir -p {quoted_dest} && cp -R {quoted_source}/. {quoted_dest}/')
+    return ' && '.join(commands)
+
+
+def quote_path_with_home(path: str) -> str:
+    """Quote a shell path while preserving leading ``$HOME`` expansion."""
+    if path == '$HOME':
+        return '"$HOME"'
+    if path.startswith('$HOME/'):
+        rest = path[len('$HOME/'):]
+        if not rest:
+            return '"$HOME"'
+        return f'"$HOME"/{shlex.quote(rest)}'
+    return shlex.quote(path)
+
+
+def _dedupe_paths(paths: Iterable[str]) -> List[str]:
+    resolved: List[str] = []
+    seen = set()
+    for path in paths:
+        if not path or path in seen:
+            continue
+        seen.add(path)
+        resolved.append(path)
+    return resolved
+
+
+__all__ = [
+    'CONFIG_SKILL_SOURCE',
+    'DEFAULT_SKILLS_INSTALL_DIR',
+    'DEFAULT_SKILLS_SANDBOX_DIR',
+    'ResolvedSkills',
+    'SkillMetadata',
+    'TASK_BUNDLED_SKILL_SOURCE',
+    'discover_skills',
+    'format_skills_prompt',
+    'install_agent_skills',
+    'install_skills_command',
+    'parse_frontmatter',
+    'quote_path_with_home',
+    'resolve_agent_skills',
+    'stage_agent_skills',
+    'skills_from_sample_metadata',
+]

evalscope/evalscope/agent/tools/bash.py

@@ -1,6 +1,6 @@
 """bash shell tool."""
 
-from typing import Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from evalscope.api.agent import AgentEnvironment
 from evalscope.api.agent.types import ExecResult
@@ -41,6 +41,56 @@ async def run_bash(call: ToolCall, env: Optional[AgentEnvironment]) -> str:
     return _format_exec_result(result)
 
 
+def apply_bash_command_timeout_defaults(
+    handlers: Dict[str, Any],
+    tools: List[ToolInfo],
+    command_timeout: Optional[float],
+) -> Tuple[Dict[str, Any], List[ToolInfo]]:
+    """Apply a native runtime default timeout to bash calls and schema."""
+    if command_timeout is None:
+        return handlers, tools
+    updated_handlers = _apply_bash_command_timeout(handlers, command_timeout)
+    updated_tools = _apply_bash_tool_timeout_default(tools, command_timeout)
+    return updated_handlers, updated_tools
+
+
+def _apply_bash_command_timeout(handlers: Dict[str, Any], command_timeout: float) -> Dict[str, Any]:
+    if 'bash' not in handlers:
+        return handlers
+
+    bash_handler = handlers['bash']
+
+    async def run_bash_with_default_timeout(call: ToolCall, env: Optional[AgentEnvironment]) -> str:
+        args = dict(call.function.arguments or {})
+        if 'timeout' not in args:
+            call = call.model_copy(
+                update={
+                    'function': call.function.model_copy(update={'arguments': {
+                        **args,
+                        'timeout': command_timeout,
+                    }})
+                }
+            )
+        return await bash_handler(call, env)
+
+    return {**handlers, 'bash': run_bash_with_default_timeout}
+
+
+def _apply_bash_tool_timeout_default(tools: List[ToolInfo], command_timeout: float) -> List[ToolInfo]:
+    updated_tools: List[ToolInfo] = []
+    for tool in tools:
+        if tool.name != 'bash':
+            updated_tools.append(tool)
+            continue
+        copied = tool.model_copy(deep=True)
+        timeout_param = copied.parameters.properties.get('timeout')
+        if timeout_param is not None:
+            timeout_param.default = command_timeout
+            timeout_param.description = f'Maximum execution time in seconds (default: {command_timeout:g}).'
+        updated_tools.append(copied)
+    return updated_tools
+
+
 def _format_exec_result(result: ExecResult) -> str:
     parts = []
     if result.stdout:
@@ -54,4 +104,4 @@ def _format_exec_result(result: ExecResult) -> str:
     return '\n'.join(parts) if parts else '(no output)'
 
 
-__all__ = ['run_bash', 'BASH_TOOL_INFO']
+__all__ = ['run_bash', 'BASH_TOOL_INFO', 'apply_bash_command_timeout_defaults']

evalscope/evalscope/api/agent/environment.py

@@ -7,6 +7,7 @@ live in ``evalscope/agent/environments/``.
 """
 
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import Dict, List, Optional
 
 from .types import ExecResult
@@ -46,6 +47,16 @@ class AgentEnvironment(ABC):
         """Release any external resources (containers, temp dirs, ...)."""
         ...
 
+    async def put_dir(self, source_dir: str | Path, target_dir: str) -> None:
+        """Copy a host directory into the environment.
+
+        Environments that do not expose a writable filesystem bridge should
+        raise ``NotImplementedError``. The default keeps older custom
+        environments source-compatible while allowing runners to probe for the
+        capability when they need host-side assets such as Agent Skills.
+        """
+        raise NotImplementedError(f'{type(self).__name__} does not support put_dir')
+
     async def __aenter__(self) -> 'AgentEnvironment':
         return self
 

evalscope/evalscope/api/agent/runner.py

@@ -44,12 +44,13 @@ def run_agent_loop(
     trace_strategy_name: Optional[str],
     trace_env_name: Optional[str],
     mcp_configs: Optional[List['MCPServerConfig']] = None,
+    close_environment: bool = True,
 ) -> AgentLoopResult:
     """Drive a single :class:`AgentLoop` to completion and return its result.
 
-    The environment (when provided) is closed in a ``finally`` block so
-    callers do not have to handle teardown themselves. ``AsyncioLoopRunner``
-    bridges the async loop into a synchronous call site.
+    The environment (when provided) is closed in a ``finally`` block by
+    default so callers do not have to handle teardown themselves.
+    ``AsyncioLoopRunner`` bridges the async loop into a synchronous call site.
 
     Args:
         model: The :class:`Model` driving generation.
@@ -66,6 +67,9 @@ def run_agent_loop(
             tools are merged into ``handlers`` / ``all_tools`` for the
             duration of the loop. Servers are spawned per sample (see
             :func:`evalscope.api.agent.mcp.resolve_mcp_tools`).
+        close_environment: Whether this helper owns and closes ``environment``.
+            Set to ``False`` when the caller needs to reuse the same
+            environment after the agent loop, for example to run a verifier.
 
     Returns:
         AgentLoopResult: Completed result with ``messages``, ``trace`` and
@@ -110,7 +114,7 @@ def run_agent_loop(
                 )
                 return await loop.run(ctx)
             finally:
-                if environment is not None:
+                if close_environment and environment is not None:
                     await environment.close()
 
     return AsyncioLoopRunner.run(_run())

evalscope/evalscope/api/agent/types.py

@@ -7,7 +7,7 @@ import from ``evalscope.api.agent`` to participate.
 """
 
 from dataclasses import dataclass, field
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional
 
 from evalscope.api.messages import ChatMessage
@@ -43,15 +43,21 @@ class BaseAgentConfig(BaseModel):
     """Free-form variant-specific options.  Native: forwarded to the
     strategy constructor.  External: forwarded to the runner constructor."""
 
+    skills_dir: Optional[str] = Field(default=None)
+    """Optional Agent Skills directory. Only agent runners consume this field."""
+
+    skill_prompt_nudge: bool = Field(default=True)
+    """Whether to add a neutral prompt nudge when skills are available."""
+
 
 class NativeAgentConfig(BaseAgentConfig):
     """AgentLoop-driven agent configuration.
 
     When carried by ``TaskConfig.agent_config``, every
     ``DefaultDataAdapter``-based benchmark routes inference through the
-    AgentLoop instead of calling ``model.generate`` once.  Individual
-    AgentAdapter subclasses (e.g. SWE-bench_Pro) ignore this global config
-    and use their own settings to avoid double wrapping.
+    AgentLoop instead of calling ``model.generate`` once.  AgentLoopAdapter
+    subclasses keep their benchmark defaults when fields are omitted and
+    accept explicitly configured strategy, tools and max_steps values.
     """
 
     mode: Literal['native'] = Field(default='native')
@@ -66,6 +72,13 @@ class NativeAgentConfig(BaseAgentConfig):
     max_steps: int = Field(default=10)
     """Hard upper bound on loop iterations."""
 
+    command_timeout: Optional[float] = Field(default=None)
+    """Default timeout in seconds for native command-style tools.
+
+    Tool calls that explicitly pass a timeout keep their own value. ``None``
+    means use each tool's built-in default.
+    """
+
     mcp_servers: List[MCPServerConfig] = Field(default_factory=list)
     """List of MCP servers spawned alongside this agent's per-sample loop.
 
@@ -76,6 +89,20 @@ class NativeAgentConfig(BaseAgentConfig):
     ``mcp_servers`` (the default) do not require ``pip install mcp``.
     """
 
+    @field_validator('max_steps')
+    @classmethod
+    def _validate_max_steps(cls, v: int) -> int:
+        if v <= 0:
+            raise ValueError('max_steps must be greater than 0.')
+        return v
+
+    @field_validator('command_timeout')
+    @classmethod
+    def _validate_command_timeout(cls, v: Optional[float]) -> Optional[float]:
+        if v is not None and v <= 0:
+            raise ValueError('command_timeout must be greater than 0.')
+        return v
+
 
 class ExecResult(BaseModel):
     """Result of executing a command in an ``AgentEnvironment``."""

evalscope/evalscope/api/benchmark/adapters/agent_adapter.py

@@ -19,6 +19,11 @@ Two extension modes are supported:
 
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
+from evalscope.agent.tools.bash import apply_bash_command_timeout_defaults
+from evalscope.api.agent import AgentLoopResult, NativeAgentConfig
+from evalscope.api.evaluator import InferenceResult
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import get_strategy, resolve_tool_infos, resolve_tools
 from .default_data_adapter import DefaultDataAdapter
 
 if TYPE_CHECKING:
@@ -52,16 +57,18 @@ class AgentLoopAdapter(AgentAdapter):
     #: ``'swe_bench_toolcall'``).
     strategy_name: str = 'function_calling'
 
-    #: Default upper bound on loop iterations per sample. Subclasses may
-    #: override either via class attribute or by pushing a ``max_steps``
-    #: entry into ``extra_params`` (read in ``__init__``).
+    #: Optional benchmark-level default timeout for bash-style tools. Explicit
+    #: ``NativeAgentConfig.command_timeout`` values take precedence.
+    command_timeout_default: Optional[float] = None
+
+    #: Default upper bound on loop iterations per sample. Subclasses override
+    #: this class attribute; users can explicitly override it through
+    #: ``NativeAgentConfig.max_steps``.
     max_steps_default: int = 30
 
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        # Allow benchmarks to expose ``max_steps`` to end users via the
-        # ``extra_params`` block of their ``BenchmarkMeta`` registration.
-        self.max_steps = int(self.extra_params.get('max_steps', self.max_steps_default))
+        self.max_steps = self.max_steps_default
 
     # ------------------------------------------------------------------
     # Build hooks
@@ -73,8 +80,6 @@ class AgentLoopAdapter(AgentAdapter):
         Default: lookup ``self.strategy_name`` in the strategy registry and
         instantiate with default parameters.
         """
-        from evalscope.api.registry import get_strategy
-
         strategy_cls = get_strategy(self.strategy_name)
         return strategy_cls()
 
@@ -90,6 +95,27 @@ class AgentLoopAdapter(AgentAdapter):
         """Return an :class:`AgentEnvironment` or ``None`` if not needed."""
         return None
 
+    def _task_sandbox_config(self) -> Dict[str, Any]:
+        """Return task-level sandbox defaults for benchmark-owned environments."""
+        if self._task_config is None or self._task_config.sandbox is None:
+            return {}
+        return dict(self._task_config.sandbox.default_config or {})
+
+    def _native_command_timeout(self) -> Optional[float]:
+        """Return the NativeAgentConfig command timeout, when explicitly configured."""
+        if self._task_config is None:
+            return None
+
+        ac = self._task_config.agent_config
+        if isinstance(ac, NativeAgentConfig):
+            return ac.command_timeout
+        return None
+
+    def _resolve_command_timeout(self, ac: Any) -> Optional[float]:
+        if isinstance(ac, NativeAgentConfig) and 'command_timeout' in ac.model_fields_set:
+            return ac.command_timeout
+        return self.command_timeout_default
+
     def build_initial_messages(self, sample: Any) -> List[Any]:
         """Return the message list the loop starts with.
 
@@ -97,12 +123,90 @@ class AgentLoopAdapter(AgentAdapter):
         :class:`ChatMessageUser` when it is a plain string, otherwise copy
         the list.
         """
-        from evalscope.api.messages import ChatMessageUser
-
         if isinstance(sample.input, list):
             return list(sample.input)
         return [ChatMessageUser(content=sample.input)]
 
+    def build_max_steps_finalization_message(self, sample: Any) -> Optional[str]:
+        """Return a no-tools finalization prompt after the loop exhausts its step budget.
+
+        The default ``None`` keeps the standard AgentLoop result. Benchmarks whose
+        official protocol requires one final model turn can override this hook.
+        """
+        return None
+
+    def should_finalize_after_max_steps(self, result: AgentLoopResult) -> bool:
+        """Return whether a max-steps result needs the optional final model turn."""
+        return not result.final_output.completion.strip()
+
+    def _maybe_run_external_agent(self, ac: Any, model: Any, sample: Any) -> Optional[InferenceResult]:
+        if ac is None or isinstance(ac, NativeAgentConfig):
+            return None
+
+        # Local import to keep the bridge stack out of the adapter's
+        # module-load-time imports (no aiohttp dependency for non-external
+        # benchmark runs).
+        from evalscope.agent.external.adapter import run_external_agent
+        from evalscope.agent.external.config import ExternalAgentConfig
+
+        if not isinstance(ac, ExternalAgentConfig):
+            return None
+
+        messages = self.build_initial_messages(sample)
+        instruction = '\n\n'.join(m.text for m in messages if getattr(m, 'text', ''))
+        return run_external_agent(
+            config=ac,
+            model=model,
+            sample=sample,
+            environment_override=self.build_environment(sample),
+            instruction_override=instruction,
+            post_run_hook=self._external_extract_prediction,
+        )
+
+    def _resolve_strategy(self, sample: Any, ac: Any) -> Any:
+        strategy = self.build_strategy(sample)
+        if not isinstance(ac, NativeAgentConfig):
+            return strategy
+
+        explicit_fields = ac.model_fields_set
+        if 'strategy' not in explicit_fields and 'kwargs' not in explicit_fields:
+            return strategy
+
+        # Keep the benchmark strategy name when the user only supplies kwargs.
+        strategy_name = ac.strategy if 'strategy' in explicit_fields else strategy.name
+        return get_strategy(strategy_name)(**ac.kwargs)
+
+    def _resolve_max_steps(self, ac: Any) -> int:
+        if isinstance(ac, NativeAgentConfig) and 'max_steps' in ac.model_fields_set:
+            return ac.max_steps
+        return self.max_steps
+
+    def _resolve_tools(self, sample: Any, ac: Any) -> tuple[Dict[str, Any], List[Any]]:
+        handlers = self.build_tools(sample)
+        all_tools = list(sample.tools or [])
+        command_timeout = self._resolve_command_timeout(ac)
+        if not isinstance(ac, NativeAgentConfig):
+            return apply_bash_command_timeout_defaults(handlers, all_tools, command_timeout)
+
+        if ac.tools:
+            configured_handlers = resolve_tools(ac.tools)
+            # Benchmark handlers win name collisions so required task semantics
+            # cannot be replaced by a global config.
+            handlers = {**configured_handlers, **handlers}
+            existing_tool_names = {tool.name for tool in all_tools}
+            for tool_info in resolve_tool_infos(ac.tools):
+                if tool_info.name not in existing_tool_names:
+                    all_tools.append(tool_info)
+                    existing_tool_names.add(tool_info.name)
+
+        return apply_bash_command_timeout_defaults(handlers, all_tools, command_timeout)
+
+    @staticmethod
+    def _resolve_mcp_configs(ac: Any) -> Optional[List['MCPServerConfig']]:
+        if isinstance(ac, NativeAgentConfig):
+            return ac.mcp_servers or None
+        return None
+
     # ------------------------------------------------------------------
     # Overridden inference hook
     # ------------------------------------------------------------------
@@ -110,11 +214,11 @@ class AgentLoopAdapter(AgentAdapter):
     def _on_inference(self, model: Any, sample: Any) -> Any:
         """Drive :class:`AgentLoop` for this sample and return the final output.
 
-        ``NativeAgentConfig.mcp_servers`` (if any) is forwarded to
-        :func:`run_agent_loop` so MCP-advertised tools merge into this
-        adapter's tool set without any benchmark-side change. Other
-        ``NativeAgentConfig`` fields are ignored — agentic benchmarks
-        are self-contained by design.
+        Benchmark defaults remain authoritative when ``agent_config`` is
+        omitted. Explicit ``NativeAgentConfig`` strategy, tools and max_steps
+        fields override or extend those defaults; MCP tools are always merged.
+        The benchmark keeps ownership of its environment so task-specific
+        mounts and sandbox contracts remain intact.
 
         :class:`ExternalAgentConfig` routes through :func:`run_external_agent`
         directly, with the adapter's :meth:`build_environment` and
@@ -122,34 +226,20 @@ class AgentLoopAdapter(AgentAdapter):
         and prompt, and :meth:`_external_extract_prediction` recovering
         the prediction artifact before the env closes.
         """
-        from evalscope.api.agent import AgentLoopResult, run_agent_loop
-        from evalscope.api.evaluator import InferenceResult
+        from evalscope.api.agent import run_agent_loop
 
         ac = self._task_config.agent_config if self._task_config is not None else None
-        mcp_configs: Optional[List['MCPServerConfig']] = None
-        if ac is not None:
-            # Local import to keep the bridge stack out of the adapter's
-            # module-load-time imports (no aiohttp dependency for non-
-            # external benchmark runs).
-            from evalscope.agent.external.adapter import run_external_agent
-            from evalscope.agent.external.config import ExternalAgentConfig
-            if isinstance(ac, ExternalAgentConfig):
-                messages = self.build_initial_messages(sample)
-                instruction = '\n\n'.join(m.text for m in messages if getattr(m, 'text', ''))
-                return run_external_agent(
-                    config=ac,
-                    model=model,
-                    sample=sample,
-                    environment_override=self.build_environment(sample),
-                    instruction_override=instruction,
-                    post_run_hook=self._external_extract_prediction,
-                )
-            # NativeAgentConfig: forward only ``mcp_servers``; benchmark
-            # adapters own their strategy / tools / max_steps.
-            mcp_configs = ac.mcp_servers or None
+        external_result = self._maybe_run_external_agent(ac, model, sample)
+        if external_result is not None:
+            return external_result
 
-        strategy = self.build_strategy(sample)
-        handlers = self.build_tools(sample)
+        strategy = self._resolve_strategy(sample, ac)
+        handlers, all_tools = self._resolve_tools(sample, ac)
+        max_steps = self._resolve_max_steps(ac)
+        mcp_configs = self._resolve_mcp_configs(ac)
+
+        if max_steps <= 0:
+            raise ValueError('AgentLoop max_steps must be greater than 0.')
         environment = self.build_environment(sample)
 
         result: AgentLoopResult = run_agent_loop(
@@ -158,14 +248,18 @@ class AgentLoopAdapter(AgentAdapter):
             handlers=handlers,
             environment=environment,
             initial_messages=self.build_initial_messages(sample),
-            all_tools=list(sample.tools or []),
-            max_steps=self.max_steps,
+            all_tools=all_tools,
+            max_steps=max_steps,
             sample_id=sample.id,
             trace_strategy_name=getattr(strategy, 'name', None),
             trace_env_name=environment.name if environment else None,
             mcp_configs=mcp_configs,
         )
 
+        finalization_prompt = self.build_max_steps_finalization_message(sample)
+        if finalization_prompt and self._reached_max_steps(result) and self.should_finalize_after_max_steps(result):
+            return self._finalize_after_max_steps(model, result, finalization_prompt)
+
         # Resolve the final prediction through the strategy → adapter hook
         # chain so benchmarks (e.g. SWE-bench) can extract a custom payload
         # like a git patch from the trajectory.
@@ -175,6 +269,63 @@ class AgentLoopAdapter(AgentAdapter):
         output.completion = final_text
         return InferenceResult(output=output, messages=result.messages, trace=result.trace)
 
+    @staticmethod
+    def _reached_max_steps(result: AgentLoopResult) -> bool:
+        if result.trace is None:
+            return False
+        from evalscope.api.agent import EventType
+        return any(
+            event.type == EventType.ERROR and event.payload.get('message') == 'max_steps_exceeded'
+            for event in result.trace.events
+        )
+
+    @staticmethod
+    def _finalize_after_max_steps(model: Any, result: AgentLoopResult, prompt: str) -> InferenceResult:
+        from evalscope.api.agent import EventType
+
+        finalization_message = ChatMessageUser(content=prompt)
+        finalization_input = list(result.messages) + [finalization_message]
+        final_output = model.generate(input=finalization_input, tools=None)
+        messages = finalization_input + [final_output.message]
+
+        step = result.trace.max_steps
+        result.trace.add_event(
+            step=step,
+            type=EventType.NUDGE,
+            message_id=finalization_message.id,
+            payload={'reason': 'max_steps_finalization'},
+        )
+        usage = None
+        if final_output.usage is not None:
+            usage = {
+                'input': final_output.usage.input_tokens,
+                'output': final_output.usage.output_tokens,
+                'total': final_output.usage.total_tokens,
+            }
+        result.trace.add_event(
+            step=step,
+            type=EventType.MODEL_GENERATE,
+            message_id=final_output.message.id,
+            token_usage=usage,
+            payload={
+                'stop_reason': final_output.stop_reason,
+                'phase': 'max_steps_finalization'
+            },
+        )
+        if final_output.completion.strip():
+            result.trace.add_event(
+                step=step,
+                type=EventType.SUBMIT,
+                message_id=final_output.message.id,
+                payload={
+                    'final_answer': final_output.completion,
+                    'phase': 'max_steps_finalization'
+                },
+            )
+            if result.trace.total_usage is not None and final_output.usage is not None:
+                result.trace.total_usage += final_output.usage
+        return InferenceResult(output=final_output, messages=messages, trace=result.trace)
+
     async def _external_extract_prediction(
         self,
         env: Any,

evalscope/evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -253,7 +253,7 @@ class DefaultDataAdapter(DataAdapter):
             sample_fields=self.record_to_sample,  # Custom sample conversion function
             filter_func=self.sample_filter,
             limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
-            repeats=self.repeats,  # Number of repetitions for each sample
+            repeats=1 if self.reformat_subset else self.repeats,  # Number of repetitions for each sample
             shuffle=self.shuffle,  # Shuffle dataset if enabled
             shuffle_choices=self.shuffle_choices,  # Shuffle choices if requested
             data_source=self.dataset_hub,  # Data source configuration
@@ -878,7 +878,9 @@ class DefaultDataAdapter(DataAdapter):
 
     def finalize(self, *args, **kwargs):
         # Finalize the evaluation process
-        self.sandbox_finalize(*args, **kwargs)
+        sandbox_finalize = getattr(self, 'sandbox_finalize', None)
+        if callable(sandbox_finalize):
+            sandbox_finalize(*args, **kwargs)
         # Release dataset memory after evaluation to avoid accumulation across benchmarks
         self.test_dataset = None
         self.fewshot_dataset = None

evalscope/evalscope/api/benchmark/benchmark.py

@@ -9,7 +9,7 @@ from evalscope.api.dataset import DatasetDict, Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.filter import FilterEnsemble, build_filter_ensemble
 from evalscope.api.metric import AggScore, SampleScore
-from evalscope.api.mixin import LLMJudgeMixin, SandboxMixin
+from evalscope.api.mixin import LLMJudgeMixin
 from evalscope.api.model import Model
 from evalscope.report import Report
 from evalscope.utils.logger import get_logger
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 logger = get_logger()
 
 
-class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC):
+class DataAdapter(LLMJudgeMixin, ABC):
     """
     Data Adapter for the benchmark.
     """

evalscope/evalscope/api/benchmark/statistics.py

@@ -546,7 +546,7 @@ class SampleExample:
         # Check if truncation is needed
         truncated_data = truncate_value(raw_data, max_length, max_list_items)
         truncated_str = str(truncated_data)
-        was_truncated = '[TRUNCATED]' in truncated_str
+        was_truncated = '[TRUNCATED' in truncated_str
 
         return cls(
             data=truncated_data,

evalscope/evalscope/api/dataset/__init__.py

@@ -1,3 +1,9 @@
+from .builder import (
+    build_dataset_dict_from_record_map,
+    build_dataset_from_records,
+    load_local_file_dataset,
+    resolve_snapshot_or_local_path,
+)
 from .dataset import Dataset, DatasetDict, FieldSpec, MemoryDataset, Sample
-from .hub import DatasetHub, download_dataset_file, load_dataset_from_hub
+from .hub import DatasetHub, download_dataset_file, download_dataset_snapshot, load_dataset_from_hub
 from .loader import DataLoader, DictDataLoader, LocalDataLoader, RemoteDataLoader

evalscope/evalscope/api/dataset/builder.py

@@ -0,0 +1,123 @@
+import copy
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Union
+
+from .dataset import DatasetDict, FieldSpec, MemoryDataset, Sample
+from .hub import DatasetHub
+from .loader import LocalDataLoader, _shuffle_in_place
+from .utils import data_to_samples, record_to_sample_fn
+
+if TYPE_CHECKING:
+    from evalscope.api.benchmark import DataAdapter
+
+
+def build_dataset_from_records(
+    records: Iterable[dict],
+    sample_fields: Union[FieldSpec, Callable],
+    *,
+    name: str,
+    location: Optional[str],
+    limit: Optional[Union[int, float]],
+    repeats: int,
+    shuffle: bool,
+    seed: Optional[int],
+    filter_func: Optional[Callable[[Sample], bool]] = None,
+    auto_id: bool = True,
+) -> MemoryDataset:
+    """Build a MemoryDataset from raw records using the standard adapter mechanics.
+
+    Note:
+        ``repeats`` duplicates each *resulting sample* (not the raw record) ``repeats``
+        times consecutively, then ``reindex`` groups them with ``group_size=repeats``.
+        This matches the single-sample-per-record adapters. If ``sample_fields`` maps one
+        record to multiple samples, the per-sample duplication order differs from record
+        level duplication, which callers relying on k-metric grouping should be aware of.
+    """
+    record_list = list(records)
+    if shuffle:
+        _shuffle_in_place(record_list, seed)
+
+    if limit is not None:
+        if isinstance(limit, float):
+            if not (0.0 <= limit <= 1.0):
+                raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
+            limit = int(len(record_list) * limit)
+        elif isinstance(limit, int) and limit < 0:
+            raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
+        record_list = record_list[:limit]
+
+    data_to_sample = record_to_sample_fn(sample_fields)
+    samples = data_to_samples(data=record_list, data_to_sample=data_to_sample)
+    if repeats > 1:
+        samples = [copy.deepcopy(sample) for sample in samples for _ in range(repeats)]
+
+    dataset = MemoryDataset(samples=samples, name=name, location=location, shuffled=shuffle)
+    if filter_func is not None:
+        dataset = dataset.filter(filter_func)
+    if auto_id:
+        dataset.reindex(group_size=repeats if repeats > 0 else 1)
+    return dataset
+
+
+def build_dataset_dict_from_record_map(
+    record_map: Dict[str, Iterable[dict]],
+    sample_fields: Union[FieldSpec, Callable],
+    *,
+    location: Optional[str],
+    limit: Optional[Union[int, float]],
+    repeats: int,
+    shuffle: bool,
+    seed: Optional[int],
+    filter_func: Optional[Callable[[Sample], bool]] = None,
+    auto_id: bool = True,
+) -> DatasetDict:
+    """Build a DatasetDict from a mapping of subset name to raw records."""
+    datasets = {}
+    for subset, records in record_map.items():
+        datasets[subset] = build_dataset_from_records(
+            records=records,
+            sample_fields=sample_fields,
+            name=subset,
+            location=location,
+            limit=limit,
+            repeats=repeats,
+            shuffle=shuffle,
+            seed=seed,
+            filter_func=filter_func,
+            auto_id=auto_id,
+        )
+    return DatasetDict(datasets)
+
+
+def resolve_snapshot_or_local_path(
+    adapter: 'DataAdapter',
+    allow_file_pattern: Optional[Union[str, List[str]]] = None,
+) -> str:
+    """Resolve an adapter dataset_id as a local path or downloaded snapshot root."""
+    return DatasetHub(
+        data_id_or_path=adapter.dataset_id,
+        data_source=adapter.dataset_hub,
+        force_redownload=adapter.force_redownload,
+        cache_dir=adapter.dataset_dir,
+    ).download_snapshot(allow_file_pattern=allow_file_pattern)
+
+
+def load_local_file_dataset(
+    adapter: 'DataAdapter',
+    dataset_path: str,
+    subset: str,
+    split: str,
+    sample_fields: Union[FieldSpec, Callable],
+    limit: Optional[Union[int, float]],
+    repeats: int,
+    shuffle: bool,
+) -> MemoryDataset:
+    """Load a local JSONL/CSV/TSV file or directory with the standard LocalDataLoader."""
+    return LocalDataLoader(
+        data_id_or_path=dataset_path,
+        split=split,
+        subset=subset,
+        sample_fields=sample_fields,
+        limit=limit,
+        repeats=repeats,
+        shuffle=shuffle,
+    ).load()

evalscope/evalscope/api/dataset/hub.py

@@ -3,7 +3,7 @@
 import inspect
 import os
 from dataclasses import dataclass
-from typing import Optional
+from typing import List, Optional, Union
 
 from evalscope.constants import HubType
 from evalscope.utils.logger import get_logger
@@ -44,6 +44,27 @@ class DatasetHub:
             cache_dir=self.cache_dir,
         )
 
+    def download_snapshot(
+        self,
+        allow_file_pattern: Optional[Union[str, List[str]]] = None,
+        ignore_file_pattern: Optional[Union[str, List[str]]] = None,
+    ) -> str:
+        return download_dataset_snapshot(
+            data_id_or_path=self.data_id_or_path,
+            data_source=self.data_source,
+            revision=self.revision,
+            force_redownload=self.force_redownload,
+            cache_dir=self.cache_dir,
+            allow_file_pattern=allow_file_pattern,
+            ignore_file_pattern=ignore_file_pattern,
+        )
+
+
+def _resolve_data_source(data_id_or_path: str, data_source: Optional[str]) -> str:
+    if data_source == HubType.LOCAL or os.path.exists(data_id_or_path):
+        return HubType.LOCAL
+    return data_source or HubType.MODELSCOPE
+
 
 def load_dataset_from_hub(
     data_id_or_path: str,
@@ -61,7 +82,7 @@ def load_dataset_from_hub(
     from modelscope import MsDataset
     from modelscope.utils.constant import DownloadMode as MSDownloadMode
 
-    data_source = data_source or HubType.MODELSCOPE
+    data_source = _resolve_data_source(data_id_or_path, data_source)
     hf_download_mode = None if not force_redownload else HFDownloadMode.FORCE_REDOWNLOAD
     ms_download_mode = None if not force_redownload else MSDownloadMode.FORCE_REDOWNLOAD
 
@@ -69,7 +90,7 @@ def load_dataset_from_hub(
         load_kwargs = dict(
             dataset_name=data_id_or_path,
             split=split,
-            subset_name=subset,
+            subset_name=subset if subset != 'default' else None,
             trust_remote_code=trust_remote,
             **kwargs,
         )
@@ -113,7 +134,16 @@ def download_dataset_file(
     cache_dir: Optional[str] = None,
 ) -> str:
     """Download or resolve a single file from a dataset hub."""
-    data_source = data_source or HubType.MODELSCOPE
+    data_source = _resolve_data_source(data_id_or_path, data_source)
+
+    if data_source == HubType.LOCAL:
+        root_dir = os.path.realpath(data_id_or_path)
+        resolved_path = os.path.realpath(os.path.join(root_dir, file_path))
+        if os.path.commonpath([root_dir, resolved_path]) != root_dir:
+            raise ValueError(f'Invalid dataset file path: {file_path}')
+        if not os.path.exists(resolved_path):
+            raise FileNotFoundError(f'Dataset file {file_path} was not found in {root_dir}.')
+        return resolved_path
 
     if data_source == HubType.HUGGINGFACE:
         from huggingface_hub import hf_hub_download
@@ -133,19 +163,60 @@ def download_dataset_file(
         download_kwargs = {'allow_file_pattern': file_path}
         if revision:
             download_kwargs['revision'] = revision
+        if cache_dir:
+            download_kwargs['cache_dir'] = cache_dir
         snapshot_dir = dataset_snapshot_download(data_id_or_path, **download_kwargs)
         resolved_path = os.path.join(snapshot_dir, file_path)
         if not os.path.exists(resolved_path):
             raise FileNotFoundError(f'Dataset file {file_path} was not found in {snapshot_dir}.')
         return resolved_path
 
+    raise ValueError(f'Unsupported dataset hub: {data_source}')
+
+
+def download_dataset_snapshot(
+    data_id_or_path: str,
+    data_source: Optional[str] = HubType.MODELSCOPE,
+    revision: Optional[str] = None,
+    force_redownload: bool = False,
+    cache_dir: Optional[str] = None,
+    allow_file_pattern: Optional[Union[str, List[str]]] = None,
+    ignore_file_pattern: Optional[Union[str, List[str]]] = None,
+) -> str:
+    """Download or resolve a dataset snapshot root from a supported hub."""
+    data_source = _resolve_data_source(data_id_or_path, data_source)
+
     if data_source == HubType.LOCAL:
-        root_dir = os.path.abspath(data_id_or_path)
-        resolved_path = os.path.abspath(os.path.join(root_dir, file_path))
-        if os.path.commonpath([root_dir, resolved_path]) != root_dir:
-            raise ValueError(f'Invalid dataset file path: {file_path}')
-        if not os.path.exists(resolved_path):
-            raise FileNotFoundError(f'Dataset file {file_path} was not found in {root_dir}.')
-        return resolved_path
+        root_dir = os.path.realpath(data_id_or_path)
+        if not os.path.isdir(root_dir):
+            raise FileNotFoundError(f'Local dataset directory was not found: {data_id_or_path}')
+        return root_dir
+
+    if data_source == HubType.HUGGINGFACE:
+        from huggingface_hub import snapshot_download
+
+        return snapshot_download(
+            repo_id=data_id_or_path,
+            repo_type='dataset',
+            revision=revision,
+            cache_dir=cache_dir,
+            force_download=force_redownload,
+            allow_patterns=allow_file_pattern,
+            ignore_patterns=ignore_file_pattern,
+        )
+
+    if data_source == HubType.MODELSCOPE:
+        from modelscope import dataset_snapshot_download
+
+        download_kwargs = {}
+        if revision:
+            download_kwargs['revision'] = revision
+        if cache_dir:
+            download_kwargs['cache_dir'] = cache_dir
+        if allow_file_pattern is not None:
+            download_kwargs['allow_file_pattern'] = allow_file_pattern
+        if ignore_file_pattern is not None:
+            download_kwargs['ignore_file_pattern'] = ignore_file_pattern
+        return dataset_snapshot_download(data_id_or_path, **download_kwargs)
 
     raise ValueError(f'Unsupported dataset hub: {data_source}')

evalscope/evalscope/api/evaluator/cache.py

@@ -274,6 +274,9 @@ class ModelResult(BaseModel):
     messages: List[ChatMessage] = []
     """Chat messages exchanged during evaluation (for conversational models)."""
 
+    agent_trace: Optional[AgentTrace] = None
+    """Structured agent trajectory, when prediction used an agent runner."""
+
     metadata: Optional[Dict[str, Any]] = None
     """Additional metadata associated with the model result."""
 
@@ -308,6 +311,7 @@ class ModelResult(BaseModel):
             model=task_state.model,
             index=task_state.sample_id,
             messages=task_state.messages,
+            agent_trace=task_state.agent_trace,
             model_output=task_state.output,
             metadata=task_state.metadata if save_metadata else {},
         )
@@ -335,13 +339,15 @@ class ModelResult(BaseModel):
         if self.metadata:
             sample.metadata.update(self.metadata)
 
-        return TaskState(
+        task_state = TaskState(
             model=self.model,
             sample=sample,
             messages=self.messages,
             output=ModelOutput.model_validate(self.model_output),
             completed=True,  # Mark as completed since it was cached
         )
+        task_state.agent_trace = self.agent_trace
+        return task_state
 
     def pretty_print(self) -> str:
         """

evalscope/evalscope/api/messages/content.py

@@ -80,8 +80,8 @@ class ContentVideo(ContentBase):
     video: str
     """Video file path, URL, or base64 encoded data URL."""
 
-    format: Literal['mp4', 'mpeg', 'mov']
-    """Format of video data ('mp4', 'mpeg', or 'mov')"""
+    format: Literal['mp4', 'mpeg', 'mov', 'avi']
+    """Format of video data ('mp4', 'mpeg', 'mov', or 'avi')"""
 
     start: Optional[float] = Field(default=None)
     """Optional start time of the relevant video segment in seconds."""

evalscope/evalscope/api/mixin/__init__.py

@@ -1,2 +1,2 @@
+from .code_execution_sandbox_mixin import CodeExecutionSandboxMixin
 from .llm_judge_mixin import LLMJudgeMixin
-from .sandbox_mixin import SandboxMixin

evalscope/evalscope/api/mixin/code_execution_sandbox_mixin.py

@@ -1,25 +1,26 @@
-"""Sandbox mixin – thin wrapper around :class:`SandboxService`.
+"""Code execution sandbox mixin backed by :class:`SandboxService`.
 
-Historically this module owned all ms_enclave integration.  The manager
-lifecycle, engine dispatch and docker image build logic have since been
-moved to :mod:`evalscope.api.sandbox`.  The mixin now only:
+This mixin is for benchmarks that execute generated code during scoring
+(``match_score`` / verifier logic). It owns a benchmark-level sandbox pool and
+does not model a full per-sample agent environment.
 
-* decides whether a benchmark wants the sandbox (``use_sandbox``);
+The mixin only:
+
+* decides whether a benchmark wants code execution sandboxing (``use_sandbox``);
 * resolves the engine + sandbox config from ``TaskConfig`` + ``BenchmarkMeta``;
 * exposes a simple ``execute_code_in_sandbox`` facade for adapters.
 """
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 
 from evalscope.api.sandbox import (
+    DockerImageSpec,
     PoolHandle,
     SandboxEngine,
     build_and_acquire_pool_sync,
-    default_docker_build_context,
-    ensure_docker_image_built,
     merge_sandbox_config_dicts,
-    normalize_docker_build_context,
+    prepare_docker_image,
     resolve_engine,
 )
 from evalscope.utils.function_utils import AsyncioLoopRunner, thread_safe
@@ -34,11 +35,11 @@ if TYPE_CHECKING:
 logger = get_logger()
 
 
-class SandboxBackend(ABC):
-    """Abstract base class for sandbox backends.
+class CodeExecutionBackend(ABC):
+    """Abstract base class for code execution sandbox backends.
 
     Kept so alternative non-ms_enclave backends can be plugged in later.
-    Today there is a single implementation (:class:`EnclaveSandboxBackend`).
+    Today there is a single implementation (:class:`EnclaveCodeExecutionBackend`).
     """
 
     def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: 'TaskConfig'):
@@ -62,21 +63,19 @@ class SandboxBackend(ABC):
         """Release any sandbox-local resources (not the service itself)."""
 
 
-class EnclaveSandboxBackend(SandboxBackend):
-    """ms_enclave-backed sandbox backend delegating to :class:`SandboxService`."""
+class EnclaveCodeExecutionBackend(CodeExecutionBackend):
+    """ms_enclave-backed code execution pool delegating to :class:`SandboxService`."""
 
     def __init__(
         self,
         benchmark_meta: 'BenchmarkMeta',
         task_config: 'TaskConfig',
-        use_custom_image: bool = False,
-        build_context_provider: Optional[Callable[[], Tuple[str, str]]] = None,
+        image_spec_provider: Optional[Callable[[], Optional[DockerImageSpec]]] = None,
     ):
         super().__init__(benchmark_meta, task_config)
         self._pool_handle: Optional[PoolHandle] = None
         self._pool_size: int = self._resolve_pool_size()
-        self._use_custom_image = use_custom_image
-        self._build_context_provider = build_context_provider
+        self._image_spec_provider = image_spec_provider
 
     def _resolve_pool_size(self) -> int:
         if not self._task_config:
@@ -96,10 +95,12 @@ class EnclaveSandboxBackend(SandboxBackend):
         manager_config = self._resolve_manager_config()
 
         if engine is SandboxEngine.DOCKER:
-            image = sandbox_cfg_dict.get('image')
-            if self._use_custom_image and image:
-                build_ctx, dockerfile = self._get_build_context()
-                ensure_docker_image_built(image, path=build_ctx, dockerfile=dockerfile, label='Sandbox image')
+            image_spec = self._image_spec_provider() if self._image_spec_provider is not None else None
+            if image_spec is not None:
+                result = prepare_docker_image(image_spec)
+                sandbox_cfg_dict = dict(sandbox_cfg_dict)
+                sandbox_cfg_dict['image'] = result.image_tag
+                logger.info(f'Sandbox image prepared: {result.image_tag} (reused={result.reused})')
 
         self._pool_handle = build_and_acquire_pool_sync(
             engine=engine,
@@ -160,13 +161,6 @@ class EnclaveSandboxBackend(SandboxBackend):
         # Manager lifecycle is owned by SandboxService; we only drop our pool reference.
         self._pool_handle = None
 
-    def _get_build_context(self) -> Tuple[str, str]:
-        if self._build_context_provider is None:
-            build_ctx, dockerfile = default_docker_build_context()
-        else:
-            build_ctx, dockerfile = self._build_context_provider()
-        return normalize_docker_build_context(build_ctx, dockerfile)
-
     # ------------------------------------------------------------------
     # Config resolution helpers
     # ------------------------------------------------------------------
@@ -183,7 +177,7 @@ class EnclaveSandboxBackend(SandboxBackend):
         return merge_sandbox_config_dicts(meta_default, task_default)
 
     # ------------------------------------------------------------------
-    # Legacy accessor used by ``SandboxMixin.sandbox_manager``
+    # Accessor used by ``CodeExecutionSandboxMixin.sandbox_manager``
     # ------------------------------------------------------------------
 
     @property
@@ -191,21 +185,21 @@ class EnclaveSandboxBackend(SandboxBackend):
         return self._pool_handle.manager if self._pool_handle else None
 
 
-class SandboxMixin:
-    """Sandbox mixin for sandboxed code execution."""
+class CodeExecutionSandboxMixin:
+    """Mixin for benchmarks that execute generated code in a sandbox pool."""
 
     def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None):
         self._benchmark_meta = benchmark_meta
         self._task_config = task_config
 
-        self._backend: Optional[SandboxBackend] = None
-        """Sandbox backend instance."""
+        self._backend: Optional[CodeExecutionBackend] = None
+        """Code execution sandbox backend instance."""
 
-        super().__init__()
+        super().__init__(benchmark_meta=benchmark_meta, task_config=task_config)
 
     @property
     def use_sandbox(self) -> bool:
-        """Return whether to use sandbox for the benchmark."""
+        """Return whether to use sandboxed code execution for the benchmark."""
         if not self._task_config or self._task_config.sandbox is None:
             return False
         return bool(self._task_config.sandbox.enabled)
@@ -213,24 +207,23 @@ class SandboxMixin:
     @property
     def sandbox_manager(self) -> Optional['SandboxManager']:
         """Get the underlying SandboxManager instance (or ``None`` if not started)."""
-        if isinstance(self._backend, EnclaveSandboxBackend):
+        if isinstance(self._backend, EnclaveCodeExecutionBackend):
             return self._backend.manager
         return None
 
-    def _get_backend(self) -> SandboxBackend:
+    def _get_backend(self) -> CodeExecutionBackend:
         if self._backend:
             return self._backend
-        self._backend = EnclaveSandboxBackend(
+        self._backend = EnclaveCodeExecutionBackend(
             self._benchmark_meta,
             self._task_config,
-            use_custom_image=bool(getattr(self, '_use_custom_image', False)),
-            build_context_provider=self.get_build_context,
+            image_spec_provider=self.get_sandbox_image_spec,
         )
         return self._backend
 
-    def get_build_context(self) -> Tuple[str, str]:
-        """Return Docker build context for benchmarks using a custom sandbox image."""
-        return default_docker_build_context()
+    def get_sandbox_image_spec(self) -> Optional[DockerImageSpec]:
+        """Return a benchmark-level Docker image spec for the code execution pool, if needed."""
+        return None
 
     @thread_safe
     def ensure_sandbox_ready(self) -> bool:

evalscope/evalscope/api/mixin/llm_judge_mixin.py

@@ -4,6 +4,7 @@ from evalscope.api.evaluator import TaskState
 from evalscope.api.metric import Score
 from evalscope.constants import JudgeStrategy
 from evalscope.metrics import LLMJudge
+from evalscope.utils.argument_utils import get_secret_value
 from evalscope.utils.logger import get_logger
 
 if TYPE_CHECKING:
@@ -18,17 +19,17 @@ class LLMJudgeMixin:
     Mixin class for LLM Judge functionality.
     """
 
+    llm_judge_default = False
+    """Whether JudgeStrategy.AUTO should use LLM judge by default."""
+
     def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None):
         self._benchmark_meta = benchmark_meta
         self._task_config = task_config
 
-        self._use_llm_judge = False
-        """Whether to use LLM as a judge"""
-
         self._llm_judge: Optional[LLMJudge] = None
         """LLM judge instance"""
 
-        super().__init__(benchmark_meta=benchmark_meta, task_config=task_config)
+        super().__init__()
 
     @property
     def llm_judge(self) -> Optional[LLMJudge]:
@@ -57,7 +58,7 @@ class LLMJudgeMixin:
         elif self.judge_strategy == JudgeStrategy.LLM_RECALL:
             return True
         elif self.judge_strategy == JudgeStrategy.AUTO:
-            return self._use_llm_judge
+            return self.llm_judge_default
         else:
             logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.')
             return False
@@ -78,7 +79,8 @@ class LLMJudgeMixin:
                     'LLM judge model arguments must be provided for LLM-based judge strategies. '
                     'Please check your task configuration.'
                 )
-            return LLMJudge(**self._task_config.judge_model_args)
+            judge_model_args = get_secret_value(self._task_config.judge_model_args)
+            return LLMJudge(**judge_model_args)
 
     def maybe_llm_match_score(
         self,

evalscope/evalscope/api/model/__init__.py

@@ -1,4 +1,4 @@
-from .generate_config import GenerateConfig
+from .generate_config import AnthropicCacheControl, GenerateConfig
 from .lazy_model import LazyModel
 from .model import Model, ModelAPI, get_model, get_model_with_task_config
 from .model_output import (

evalscope/evalscope/api/model/generate_config.py

@@ -1,8 +1,9 @@
 # flake8: noqa: E501
 from copy import deepcopy
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator, model_validator
 from typing import Any, Dict, List, Literal, Optional
 
+from evalscope.utils.argument_utils import secretize_auth_headers
 from evalscope.utils.json_schema import JSONSchema
 
 
@@ -23,6 +24,18 @@ class ResponseSchema(BaseModel):
     OpenAI and Mistral only."""
 
 
+class AnthropicCacheControl(BaseModel):
+    """Anthropic prompt cache control."""
+
+    model_config = {'extra': 'forbid'}
+
+    type: Literal['ephemeral'] = Field(default='ephemeral')
+    """Anthropic cache type."""
+
+    ttl: Optional[Literal['5m', '1h']] = Field(default=None)
+    """Optional cache time-to-live."""
+
+
 class GenerateConfig(BaseModel):
     """Model generation options."""
     model_config = {'extra': 'allow'}
@@ -102,6 +115,16 @@ class GenerateConfig(BaseModel):
     reasoning_tokens: Optional[int] = Field(default=None)
     """Maximum number of tokens to use for reasoning. Anthropic Claude models only."""
 
+    anthropic_cache_control: Optional[AnthropicCacheControl] = Field(default=None)
+    """Anthropic prompt cache control. Set to ``{"type": "ephemeral"}`` to enable prompt caching."""
+
+    anthropic_cache_strategy: Literal['evaluation', 'recent_messages'] = Field(default='evaluation')
+    """Anthropic prompt cache breakpoint placement strategy.
+
+    ``evaluation`` caches stable evaluation prefixes such as tools, system prompts, and few-shot examples.
+    ``recent_messages`` caches stable agent prefixes and the growing multi-turn conversation history.
+    """
+
     reasoning_summary: Optional[Literal['concise', 'detailed', 'auto']] = Field(default=None)
     """Provide summary of reasoning steps (defaults to no summary). Use 'auto' to access the most detailed summarizer available for the current model. OpenAI reasoning models only."""
 
@@ -132,7 +155,7 @@ class GenerateConfig(BaseModel):
     extra_query: Optional[Dict[str, Any]] = Field(default=None)
     """Extra query parameters to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
 
-    extra_headers: Optional[Dict[str, str]] = Field(default=None)
+    extra_headers: Optional[Dict[str, Any]] = Field(default=None)
     """Extra headers to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
 
     height: Optional[int] = Field(default=None)
@@ -147,6 +170,21 @@ class GenerateConfig(BaseModel):
     guidance_scale: Optional[float] = Field(default=None)
     """Guidance scale for image generation model only"""
 
+    @model_validator(mode='before')
+    @classmethod
+    def reject_legacy_anthropic_cache_config(cls, data: Any) -> Any:
+        if isinstance(data, dict) and 'anthropic_content_cache_control' in data:
+            raise ValueError(
+                '`anthropic_content_cache_control` has been replaced by `anthropic_cache_control`. '
+                'Use `anthropic_cache_strategy` to choose breakpoint placement.'
+            )
+        return data
+
+    @field_validator('extra_headers', mode='after')
+    @classmethod
+    def _validate_extra_headers(cls, value: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        return secretize_auth_headers(value)
+
     def merge(self, other: 'GenerateConfig') -> 'GenerateConfig':
         """Merge another model configuration into this one.
 

evalscope/evalscope/api/model/model.py

@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, Generator, List, Literal, Optional,
 from evalscope.api.messages import ChatMessage, ChatMessageAssistant, ChatMessageSystem, ChatMessageUser
 from evalscope.api.registry import get_model_api
 from evalscope.api.tool import ToolChoice, ToolFunction, ToolInfo
-from evalscope.utils import get_logger
+from evalscope.utils import get_logger, get_secret_value
 from evalscope.utils.function_utils import thread_safe
 from .generate_config import GenerateConfig
 from .model_output import ModelOutput
@@ -305,10 +305,7 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
     model = task_config.model
     eval_type = task_config.eval_type
     base_url = task_config.api_url
-    api_key = task_config.api_key
-    # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used.
-    if api_key == 'EMPTY':
-        api_key = None
+    api_key = get_secret_value(task_config.api_key)
     config = task_config.generation_config
     model_args = task_config.model_args or {}
 
@@ -364,6 +361,10 @@ def get_model(
         if cached is not None:
             return cached
 
+    # TaskConfig defaults api_key to 'EMPTY'; treat it as unset so env vars can be used.
+    if api_key == 'EMPTY':
+        api_key = None
+
     logger.info(
         f'Creating model {model} with eval_type={eval_type} '
         f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}'

evalscope/evalscope/api/sandbox/__init__.py

@@ -1,7 +1,7 @@
 """Unified sandbox service layer for evalscope.
 
 This package consolidates the ms_enclave integration used by both
-:class:`evalscope.api.mixin.sandbox_mixin.SandboxMixin` (pool-based execution
+:class:`evalscope.api.mixin.code_execution_sandbox_mixin.CodeExecutionSandboxMixin` (pool-based execution
 for code benchmarks) and the Agent environment (per-sample containers).
 
 Public surface:
@@ -21,6 +21,13 @@ from .config_builder import (
     normalize_docker_build_context,
     should_build_docker_image,
 )
+from .docker_image import (
+    DockerImageBuilder,
+    DockerImageResult,
+    DockerImageSpec,
+    hash_build_context,
+    prepare_docker_image,
+)
 from .engine import SandboxEngine, get_enclave_types, resolve_engine
 from .service import (
     PoolHandle,
@@ -39,12 +46,17 @@ __all__ = [
     'build_and_acquire_pool_sync',
     'build_docker_image',
     'build_sandbox_config',
+    'DockerImageBuilder',
+    'DockerImageResult',
+    'DockerImageSpec',
     'default_docker_build_context',
     'ensure_docker_image_built',
     'get_enclave_types',
     'get_sandbox_service',
+    'hash_build_context',
     'merge_sandbox_config_dicts',
     'normalize_docker_build_context',
+    'prepare_docker_image',
     'resolve_engine',
     'shutdown_sandbox_service',
     'should_build_docker_image',

evalscope/evalscope/api/sandbox/docker_image.py

@@ -0,0 +1,144 @@
+"""Small Docker image build/reuse helper.
+
+This module is intentionally independent from the ms_enclave sandbox service:
+benchmarks can use it to prepare local images, then pass the resulting tag to
+their existing sandbox/environment layer.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+from pydantic import BaseModel, Field
+from typing import Any, Dict, List
+
+from evalscope.utils.logger import get_logger
+from .config_builder import build_docker_image, normalize_docker_build_context, should_build_docker_image
+
+logger = get_logger()
+
+
+class DockerImageSpec(BaseModel):
+    """Description of a local Docker build."""
+
+    name_prefix: str
+    context_dir: str
+    dockerfile: str = 'Dockerfile'
+    build_args: Dict[str, str] = Field(default_factory=dict)
+    cache_key_parts: List[str] = Field(default_factory=list)
+    force_rebuild: bool = False
+
+
+class DockerImageResult(BaseModel):
+    """Result returned by :class:`DockerImageBuilder`."""
+
+    image_tag: str
+    reused: bool
+    context_hash: str
+
+
+class DockerImageBuilder:
+    """Build or reuse a local Docker image tagged from a content hash."""
+
+    builder_version: str = 'v1'
+
+    def build_or_reuse(self, spec: DockerImageSpec) -> DockerImageResult:
+        context_dir, dockerfile = normalize_docker_build_context(spec.context_dir, spec.dockerfile)
+        context_hash = hash_build_context(
+            context_dir,
+            cache_key_parts=[self.builder_version, *spec.cache_key_parts, *_build_args_cache_parts(spec.build_args)],
+        )
+        image_tag = f'{_sanitize_tag_part(spec.name_prefix)}:{context_hash[:24]}'
+        should_build = spec.force_rebuild or should_build_docker_image(image_tag)
+        if should_build:
+            logger.info(f'Docker image {image_tag!r} not found or rebuild requested. Building from {context_dir} ...')
+            build_docker_image_with_args(
+                image=image_tag,
+                path=context_dir,
+                dockerfile=dockerfile,
+                build_args=spec.build_args,
+            )
+            logger.info(f'Docker image built: {image_tag}')
+        return DockerImageResult(image_tag=image_tag, reused=not should_build, context_hash=context_hash)
+
+
+def prepare_docker_image(spec: DockerImageSpec, *, builder: DockerImageBuilder | None = None) -> DockerImageResult:
+    """Prepare a Docker image from ``spec`` using the shared local builder."""
+    return (builder or DockerImageBuilder()).build_or_reuse(spec)
+
+
+def hash_build_context(context_dir: str, *, cache_key_parts: List[str] | None = None) -> str:
+    """Return a stable hash for regular files under ``context_dir``."""
+    root = Path(context_dir)
+    digest = hashlib.sha256()
+    for part in cache_key_parts or []:
+        digest.update(b'part\0')
+        digest.update(str(part).encode('utf-8'))
+        digest.update(b'\0')
+    for file_path in sorted(path for path in root.rglob('*') if path.is_file() and not path.is_symlink()):
+        rel = file_path.relative_to(root).as_posix()
+        digest.update(b'file\0')
+        digest.update(rel.encode('utf-8'))
+        digest.update(b'\0')
+        with file_path.open('rb') as f:
+            for chunk in iter(lambda: f.read(1024 * 1024), b''):
+                digest.update(chunk)
+        try:
+            mode = file_path.stat().st_mode & 0o777
+        except OSError:
+            mode = 0
+        digest.update(f'\0mode:{mode:o}'.encode('ascii'))
+    return digest.hexdigest()
+
+
+def build_docker_image_with_args(
+    image: str,
+    path: str,
+    dockerfile: str = 'Dockerfile',
+    build_args: Dict[str, str] | None = None,
+) -> Any:
+    """Build a Docker image with optional build args."""
+    if not build_args:
+        return build_docker_image(image=image, path=path, dockerfile=dockerfile)
+
+    from docker.client import DockerClient
+
+    docker_client = DockerClient.from_env()
+    build_logs = docker_client.images.build(
+        path=path,
+        dockerfile=dockerfile,
+        tag=image,
+        rm=True,
+        buildargs=build_args,
+    )
+    for log in build_logs[1]:
+        if 'stream' in log:
+            logger.info(log['stream'].strip())
+        elif 'error' in log:
+            logger.error(log['error'])
+    return build_logs[0]
+
+
+def _sanitize_tag_part(value: str) -> str:
+    chars = []
+    for ch in value.lower():
+        if ch.isalnum() or ch in '._-/':
+            chars.append(ch)
+        else:
+            chars.append('-')
+    text = ''.join(chars).strip('.-/')
+    return text or 'evalscope-image'
+
+
+def _build_args_cache_parts(build_args: Dict[str, str]) -> List[str]:
+    return [f'build_arg:{key}={value}' for key, value in sorted(build_args.items())]
+
+
+__all__ = [
+    'DockerImageBuilder',
+    'DockerImageResult',
+    'DockerImageSpec',
+    'hash_build_context',
+    'prepare_docker_image',
+]

evalscope/evalscope/api/sandbox/service.py

@@ -2,7 +2,7 @@
 
 Unifies the two historical code paths:
 
-* ``SandboxMixin.EnclaveSandboxBackend`` – one manager per benchmark, pooled.
+* ``CodeExecutionSandboxMixin.EnclaveCodeExecutionBackend`` – one manager per benchmark, pooled.
 * ``EnclaveAgentEnvironment`` – one manager per process, per-sample containers.
 
 Both are now thin wrappers around :class:`SandboxService`.  The service
@@ -17,6 +17,7 @@ import asyncio
 import atexit
 import json
 import threading
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from evalscope.utils.function_utils import AsyncioLoopRunner
@@ -73,6 +74,12 @@ class SandboxHandle:
             raise RuntimeError('SandboxHandle already closed')
         return await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
 
+    async def put_dir(self, source_dir: str | Path, target_dir: str) -> bool:
+        """Copy a host directory into the sandbox via ms_enclave SandboxManager."""
+        if self._sandbox_id is None:
+            raise RuntimeError('SandboxHandle already closed')
+        return await self._manager.put_dir(self._sandbox_id, source_dir, target_dir)
+
     async def close(self) -> None:
         if self._sandbox_id is None:
             return
@@ -197,7 +204,7 @@ class SandboxService:
         return SandboxManagerFactory.create_manager(**manager_config)
 
     # ------------------------------------------------------------------
-    # Public APIs: pooled (SandboxMixin) and per-sample (Agent env)
+    # Public APIs: pooled (CodeExecutionSandboxMixin) and per-sample (Agent env)
     # ------------------------------------------------------------------
 
     async def acquire_pool(
@@ -241,6 +248,14 @@ class SandboxService:
                 logger.info('SandboxService: manager stopped.')
             except Exception as exc:
                 logger.warning(f'SandboxService: error stopping manager: {exc}')
+                cleanup_all = getattr(manager, 'cleanup_all_sandboxes', None)
+                if not callable(cleanup_all):
+                    continue
+                try:
+                    await cleanup_all()
+                    logger.info('SandboxService: fallback sandbox cleanup completed.')
+                except Exception as cleanup_exc:
+                    logger.warning(f'SandboxService: fallback sandbox cleanup failed: {cleanup_exc}')
 
     def shutdown_all(self) -> None:
         """Synchronous wrapper around :meth:`shutdown_all_async`."""
@@ -285,7 +300,7 @@ def shutdown_sandbox_service() -> None:
 
 
 # ---------------------------------------------------------------------------
-# Convenience helpers used by SandboxMixin / EnclaveAgentEnvironment
+# Convenience helpers used by CodeExecutionSandboxMixin / EnclaveAgentEnvironment
 # ---------------------------------------------------------------------------
 
 
@@ -295,7 +310,7 @@ def build_and_acquire_pool_sync(
     sandbox_config_dict: Optional[Dict[str, Any]],
     manager_config: Optional[Dict[str, Any]] = None,
 ) -> PoolHandle:
-    """Synchronous helper for :class:`SandboxMixin`.
+    """Synchronous helper for :class:`CodeExecutionSandboxMixin`.
 
     Combines :func:`build_sandbox_config` and :meth:`SandboxService.acquire_pool`
     and drives them through the shared :class:`AsyncioLoopRunner`.

evalscope/evalscope/benchmarks/_meta/a_okvqa.json

@@ -158,4 +158,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.179469",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/aa_lcr.json

@@ -82,4 +82,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.178244",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/acebench.json

@@ -433,4 +433,4 @@
   },
   "updated_at": "2026-06-01T15:05:41.862420",
   "translation_updated_at": "2026-06-01T15:06:01"
-}
+}

evalscope/evalscope/benchmarks/_meta/ai2d.json

@@ -148,4 +148,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.180631",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/aime24.json

@@ -74,4 +74,4 @@
   },
   "updated_at": "2026-03-16T17:43:27.479633",
   "translation_updated_at": "2026-03-16T17:46:34Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/aime25.json

@@ -74,4 +74,4 @@
   },
   "updated_at": "2026-03-16T17:43:27.477945",
   "translation_updated_at": "2026-03-16T17:46:34Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/aime26.json

@@ -74,4 +74,4 @@
   },
   "updated_at": "2026-03-16T17:43:27.476270",
   "translation_updated_at": "2026-03-16T17:46:34Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/air_bench_chat.json

@@ -334,4 +334,4 @@
   },
   "updated_at": "2026-05-18T10:23:16.641619",
   "translation_updated_at": "2026-05-18T10:23:22"
-}
+}

evalscope/evalscope/benchmarks/_meta/air_bench_foundation.json

@@ -808,4 +808,4 @@
   },
   "updated_at": "2026-05-18T10:23:16.650501",
   "translation_updated_at": "2026-05-18T10:23:22"
-}
+}

evalscope/evalscope/benchmarks/_meta/alpaca_eval.json

@@ -74,4 +74,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.181393",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/amc.json

@@ -99,4 +99,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.181373",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/anat_em.json

@@ -99,4 +99,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.334291",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/arc.json

@@ -89,4 +89,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.181351",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/arc_agi_2.json

@@ -0,0 +1,76 @@
+{
+  "meta": {
+    "pretty_name": "ARC-AGI-2",
+    "dataset_id": "evalscope/arc-agi-2",
+    "paper_url": null,
+    "tags": [
+      "Reasoning"
+    ],
+    "metrics": [
+      "acc"
+    ],
+    "few_shot_num": 0,
+    "eval_split": "test",
+    "train_split": "",
+    "subset_list": [
+      "default"
+    ],
+    "description": "\n## Overview\n\nARC-AGI-2 (Abstraction and Reasoning Corpus for Artificial General Intelligence 2) is a benchmark designed to measure an AI system's ability to efficiently acquire new skills on-the-fly, using only a handful of demonstrations. It evaluates abstract reasoning and pattern recognition through grid transformation tasks.\n\n## Task Description\n\n- **Task Type**: Abstract Reasoning / Pattern Recognition\n- **Input**: A series of input-output grid pairs (demonstrations) followed by a test input grid\n- **Output**: The predicted output grid matching the inferred transformation rule\n- **Grid Format**: 2D arrays of integers (0-9), variable sizes (up to 30x30)\n\n## Key Features\n\n- 1,000 public training tasks and 120 public evaluation tasks\n- Each task provides 2-10 demonstration input/output pairs\n- Models must infer the transformation rule from demonstrations\n- Tests abstract reasoning without reliance on learned knowledge\n- Pixel-perfect output required (exact grid match)\n\n## Evaluation Notes\n\n- Scoring is based on **exact grid match** (shape and all values must be identical)\n- Models must output the grid as a JSON 2D array\n- Zero-shot evaluation (demonstrations are provided within each task)\n- Designed to be solvable by humans but challenging for AI\n",
+    "prompt_template": "{question}",
+    "system_prompt": "You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.",
+    "few_shot_prompt_template": "",
+    "aggregation": "mean_and_pass_hat_k",
+    "extra_params": {},
+    "sandbox_config": {},
+    "category": "llm"
+  },
+  "statistics": {
+    "total_samples": 120,
+    "subset_stats": [
+      {
+        "name": "default",
+        "sample_count": 120,
+        "prompt_length_mean": 8026.79,
+        "prompt_length_min": 2437,
+        "prompt_length_max": 25471,
+        "prompt_length_std": 4038.76,
+        "target_length_mean": 1328.32
+      }
+    ],
+    "prompt_length": {
+      "mean": 8026.79,
+      "min": 2437,
+      "max": 25471,
+      "std": 4038.76
+    },
+    "target_length_mean": 1328.32,
+    "computed_at": "2026-07-03T17:22:52.935931"
+  },
+  "sample_example": {
+    "data": {
+      "input": [
+        {
+          "id": "7b97143f",
+          "content": "You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid."
+        },
+        {
+          "id": "bdaa2b22",
+          "content": "You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\n\nExamples:\nExample 1:\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else."
+        }
+      ],
+      "target": "[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]",
+      "id": 0,
+      "group_id": 0
+    },
+    "subset": "default",
+    "truncated": false
+  },
+  "readme": {
+    "en": "# ARC-AGI-2\n\n\n## Overview\n\nARC-AGI-2 (Abstraction and Reasoning Corpus for Artificial General Intelligence 2) is a benchmark designed to measure an AI system's ability to efficiently acquire new skills on-the-fly, using only a handful of demonstrations. It evaluates abstract reasoning and pattern recognition through grid transformation tasks.\n\n## Task Description\n\n- **Task Type**: Abstract Reasoning / Pattern Recognition\n- **Input**: A series of input-output grid pairs (demonstrations) followed by a test input grid\n- **Output**: The predicted output grid matching the inferred transformation rule\n- **Grid Format**: 2D arrays of integers (0-9), variable sizes (up to 30x30)\n\n## Key Features\n\n- 1,000 public training tasks and 120 public evaluation tasks\n- Each task provides 2-10 demonstration input/output pairs\n- Models must infer the transformation rule from demonstrations\n- Tests abstract reasoning without reliance on learned knowledge\n- Pixel-perfect output required (exact grid match)\n\n## Evaluation Notes\n\n- Scoring is based on **exact grid match** (shape and all values must be identical)\n- Models must output the grid as a JSON 2D array\n- Zero-shot evaluation (demonstrations are provided within each task)\n- Designed to be solvable by humans but challenging for AI\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `arc_agi_2` |\n| **Dataset ID** | [evalscope/arc-agi-2](https://modelscope.cn/datasets/evalscope/arc-agi-2/summary) |\n| **Paper** | N/A |\n| **Tags** | `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n| **Aggregation** | `mean_and_pass_hat_k` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 120 |\n| Prompt Length (Mean) | 8026.79 chars |\n| Prompt Length (Min/Max) | 2437 / 25471 chars |\n\n## Sample Example\n\n**Subset**: `default`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"7b97143f\",\n      \"content\": \"You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\"\n    },\n    {\n      \"id\": \"bdaa2b22\",\n      \"content\": \"You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\\n\\nExamples:\\nExample 1:\\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\\n\\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else.\"\n    }\n  ],\n  \"target\": \"[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]\",\n  \"id\": 0,\n  \"group_id\": 0\n}\n```\n\n## Prompt Template\n\n**System Prompt:**\n```text\nYou are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\n```\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets arc_agi_2 \\\n    --limit 10  # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['arc_agi_2'],\n    limit=10,  # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
+    "zh": "# ARC-AGI-2\n\n\n## 概述\n\nARC-AGI-2（面向人工通用智能的抽象与推理语料库 2）是一个旨在衡量 AI 系统能否仅凭少量示例就高效地即时掌握新技能的基准测试。它通过网格变换任务来评估模型的抽象推理和模式识别能力。\n\n## 任务描述\n\n- **任务类型**：抽象推理 / 模式识别\n- **输入**：一系列输入-输出网格对（示例），后跟一个测试输入网格\n- **输出**：根据推断出的变换规则生成的预测输出网格\n- **网格格式**：二维整数数组（0-9），尺寸可变（最大 30x30）\n\n## 主要特点\n\n- 包含 1,000 个公开训练任务和 120 个公开评估任务\n- 每个任务提供 2-10 个示例输入/输出对\n- 模型必须从示例中推断出变换规则\n- 测试抽象推理能力，不依赖于已学习的知识\n- 要求输出像素级精确（必须完全匹配目标网格）\n\n## 评估说明\n\n- 评分基于**精确网格匹配**（形状和所有数值必须完全一致）\n- 模型必须以 JSON 二维数组格式输出网格\n- 零样本评估（每个任务内提供示例）\n- 设计为人类可解但对 AI 具有挑战性\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `arc_agi_2` |\n| **数据集ID** | [evalscope/arc-agi-2](https://modelscope.cn/datasets/evalscope/arc-agi-2/summary) |\n| **论文** | N/A |\n| **标签** | `Reasoning` |\n| **指标** | `acc` |\n| **默认样本数** | 0-shot |\n| **评估划分** | `test` |\n| **聚合方式** | `mean_and_pass_hat_k` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 120 |\n| 提示词长度（平均） | 8026.79 字符 |\n| 提示词长度（最小/最大） | 2437 / 25471 字符 |\n\n## 样例示例\n\n**子集**: `default`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"7b97143f\",\n      \"content\": \"You are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\"\n    },\n    {\n      \"id\": \"bdaa2b22\",\n      \"content\": \"You are given a series of input-output grid pairs as examples. Each grid is a 2D array of integers (0-9). Study the pattern in the examples, then predict the output for the test input.\\n\\nExamples:\\nExample 1:\\nInput: [[0, 0, 0, 0, 0, 0, 0, 0, 0, ... [TRUNCATED 7482 chars] ... 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\\n\\nProvide the output grid as a JSON 2D array. Only output the JSON array, nothing else.\"\n    }\n  ],\n  \"target\": \"[[8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8], [8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0], [8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 8, 8, 8, 8, 0], [ ... [TRUNCATED 1596 chars] ... ], [8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0, 8, 0], [8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 0, 8, 0, 8, 8, 8, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 0]]\",\n  \"id\": 0,\n  \"group_id\": 0\n}\n```\n\n## 提示模板\n\n**系统提示**：\n```text\nYou are an expert at abstract reasoning and pattern recognition. Given input-output grid pairs as examples, you must figure out the transformation rule and apply it to a new test input to produce the correct output grid.\n```\n\n**提示模板**：\n```text\n{question}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets arc_agi_2 \\\n    --limit 10  # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['arc_agi_2'],\n    limit=10,  # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
+    "content_hash": "5946771be95fe7121f09dff910dd6da0",
+    "needs_translation": false
+  },
+  "updated_at": "2026-07-06T14:05:00.208824",
+  "translation_updated_at": "2026-07-06T14:05:25"
+}

evalscope/evalscope/benchmarks/_meta/arena_hard.json

@@ -73,4 +73,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.190759",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/arxivmath.json

@@ -112,4 +112,4 @@
   },
   "updated_at": "2026-07-03T16:31:10.439972",
   "translation_updated_at": "2026-07-03T16:31:16"
-}
+}

evalscope/evalscope/benchmarks/_meta/arxivrollbench.json

@@ -792,4 +792,4 @@
   },
   "updated_at": "2026-05-26T16:07:01.880371",
   "translation_updated_at": "2026-05-26T16:07:57"
-}
+}

evalscope/evalscope/benchmarks/_meta/arxivrollbench_full.json

@@ -792,4 +792,4 @@
   },
   "updated_at": "2026-05-26T16:16:11.135223",
   "translation_updated_at": "2026-05-26T16:18:54"
-}
+}

evalscope/evalscope/benchmarks/_meta/baby_vision.json

@@ -283,4 +283,4 @@
   },
   "updated_at": "2026-07-03T11:09:47.265495",
   "translation_updated_at": "2026-07-03T16:19:29"
-}
+}

evalscope/evalscope/benchmarks/_meta/bbh.json

@@ -333,4 +333,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.191108",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/bc2gm.json

@@ -123,4 +123,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.338567",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/bc4chemd.json

@@ -135,4 +135,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.339790",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/bc5cdr.json

@@ -123,4 +123,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.341754",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/bfcl_v3.json

@@ -336,4 +336,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.200319",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/bfcl_v4.json

@@ -332,4 +332,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.199735",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/biomix_qa.json

@@ -79,4 +79,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.192148",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/blink.json

@@ -671,4 +671,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.192814",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/broad_twitter_corpus.json

@@ -111,4 +111,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.340322",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/browsecomp.json

@@ -58,4 +58,4 @@
   },
   "updated_at": "2026-06-17T22:07:18.747702",
   "translation_updated_at": "2026-06-17T22:07:21"
-}
+}

evalscope/evalscope/benchmarks/_meta/cc_bench.json

@@ -153,4 +153,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.299360",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/ceval.json

@@ -592,4 +592,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.194461",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/chartqa.json

@@ -185,4 +185,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.195098",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/charxiv.json

@@ -191,4 +191,4 @@
   },
   "updated_at": "2026-07-03T11:09:49.196684",
   "translation_updated_at": "2026-07-03T16:19:29"
-}
+}

evalscope/evalscope/benchmarks/_meta/chinese_simpleqa.json

@@ -129,4 +129,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.200945",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/cl_bench.json

@@ -93,4 +93,4 @@
   },
   "updated_at": "2026-02-07T10:36:33.238019",
   "translation_updated_at": "2026-02-07T10:36:51Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/claw_eval.json

@@ -0,0 +1,112 @@
+{
+  "meta": {
+    "pretty_name": "Claw-Eval",
+    "dataset_id": "claw-eval/Claw-Eval",
+    "paper_url": null,
+    "tags": [
+      "Agent",
+      "MultiModal",
+      "MultiTurn"
+    ],
+    "metrics": [
+      "avg_score",
+      "pass_at_k",
+      "pass_hat_k",
+      "error_rate"
+    ],
+    "few_shot_num": 0,
+    "eval_split": "test",
+    "train_split": "",
+    "subset_list": [
+      "general",
+      "multimodal",
+      "multi_turn"
+    ],
+    "description": "\n## Overview\n\nClaw-Eval evaluates assistant agents on realistic personal-assistant workflows that require tool use, file and fixture\naccess, multimodal inputs, and simulated user interactions. EvalScope runs the pinned official Claw-Eval Python runner,\nDocker sandbox, and graders while exposing each Claw-Eval task as a normal EvalScope sample for caching, repeats,\nparallel execution, reporting, and dashboard trace review.\n\n## Task Description\n\n- **Task Type**: Agentic personal-assistant tasks with tool use, sandbox files, multimodal fixtures, and optional\n  simulated user turns.\n- **Dataset**: `claw-eval/Claw-Eval` on ModelScope.\n- **Subsets**: `general`, `multimodal`, and `multi_turn`; the current ModelScope manifest contains 300 tasks\n  (161 general, 101 multimodal, and 38 multi_turn). Use `subset_list` to select subsets.\n- **Output**: Official Claw-Eval scores and JSONL traces, EvalScope sample-level reviews, grouped summary metrics, and\n  dashboard-rendered agent traces.\n\n## Evaluation Notes\n\n- Requires Python 3.11+ and the official package installed from the pinned source commit:\n  `pip install \"claw-eval[sandbox,mock,web] @\n  git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`.\n- The installed package provides the Claw-Eval runner APIs. EvalScope also caches the same pinned source archive because\n  `tasks/` and `Dockerfile.agent` are runtime assets, then loads the task manifest and fixtures from ModelScope.\n- Full fixtures are downloaded from ModelScope (`data/fixtures.tar.gz`) and linked into the official task tree before\n  execution. The archive is large; use `limit` or `extra_params.task_ids` for smoke runs.\n- Each selected Claw-Eval task is one EvalScope sample. Official scoring runs once per sample; use EvalScope `repeats`\n  for repeated trials per task and `eval_batch_size` for task-level worker concurrency.\n- Claw-Eval runs with the official Docker sandbox image. If `claw-eval-agent:latest` is missing locally, EvalScope\n  builds it automatically from the cached official `Dockerfile.agent`. The first run can be slow.\n- EvalScope `use_cache` resumes completed task-level samples. Claw-Eval trace JSONL files are stored under\n  `outputs/.../claw_eval/<split>/traces` and converted to EvalScope agent traces for dashboard visualization.\n",
+    "prompt_template": "{question}",
+    "system_prompt": "",
+    "few_shot_prompt_template": "",
+    "aggregation": "mean",
+    "extra_params": {
+      "task_ids": {
+        "type": "list",
+        "description": "Optional exact Claw-Eval task ids to run after split filtering.",
+        "value": []
+      }
+    },
+    "sandbox_config": {},
+    "category": "agent"
+  },
+  "statistics": {
+    "total_samples": 300,
+    "subset_stats": [
+      {
+        "name": "general",
+        "sample_count": 161,
+        "prompt_length_mean": 46.47,
+        "prompt_length_min": 34,
+        "prompt_length_max": 58,
+        "prompt_length_std": 4.87,
+        "target_length_mean": null
+      },
+      {
+        "name": "multimodal",
+        "sample_count": 101,
+        "prompt_length_mean": 49.75,
+        "prompt_length_min": 30,
+        "prompt_length_max": 60,
+        "prompt_length_std": 5.97,
+        "target_length_mean": null
+      },
+      {
+        "name": "multi_turn",
+        "sample_count": 38,
+        "prompt_length_mean": 44.79,
+        "prompt_length_min": 35,
+        "prompt_length_max": 51,
+        "prompt_length_std": 3.68,
+        "target_length_mean": null
+      }
+    ],
+    "prompt_length": {
+      "mean": 47.36,
+      "min": 30,
+      "max": 60,
+      "std": 5.43
+    },
+    "target_length_mean": null,
+    "computed_at": "2026-07-14T16:59:09.900658"
+  },
+  "sample_example": {
+    "data": {
+      "input": [
+        {
+          "id": "a34b7f6b",
+          "content": "Run Claw-Eval task T001zh_email_triage."
+        }
+      ],
+      "target": "",
+      "id": 0,
+      "group_id": 0,
+      "subset_key": "general",
+      "metadata": {
+        "task_id": "T001zh_email_triage",
+        "split": "general",
+        "task_name": "",
+        "difficulty": "",
+        "dataset_id": "claw-eval/Claw-Eval",
+        "dataset_hub": "modelscope"
+      }
+    },
+    "subset": "general",
+    "truncated": false
+  },
+  "readme": {
+    "en": "# Claw-Eval\n\n\n## Overview\n\nClaw-Eval evaluates assistant agents on realistic personal-assistant workflows that require tool use, file and fixture\naccess, multimodal inputs, and simulated user interactions. EvalScope runs the pinned official Claw-Eval Python runner,\nDocker sandbox, and graders while exposing each Claw-Eval task as a normal EvalScope sample for caching, repeats,\nparallel execution, reporting, and dashboard trace review.\n\n## Task Description\n\n- **Task Type**: Agentic personal-assistant tasks with tool use, sandbox files, multimodal fixtures, and optional\n  simulated user turns.\n- **Dataset**: `claw-eval/Claw-Eval` on ModelScope.\n- **Subsets**: `general`, `multimodal`, and `multi_turn`; the current ModelScope manifest contains 300 tasks\n  (161 general, 101 multimodal, and 38 multi_turn). Use `subset_list` to select subsets.\n- **Output**: Official Claw-Eval scores and JSONL traces, EvalScope sample-level reviews, grouped summary metrics, and\n  dashboard-rendered agent traces.\n\n## Evaluation Notes\n\n- Requires Python 3.11+ and the official package installed from the pinned source commit:\n  `pip install \"claw-eval[sandbox,mock,web] @\n  git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`.\n- The installed package provides the Claw-Eval runner APIs. EvalScope also caches the same pinned source archive because\n  `tasks/` and `Dockerfile.agent` are runtime assets, then loads the task manifest and fixtures from ModelScope.\n- Full fixtures are downloaded from ModelScope (`data/fixtures.tar.gz`) and linked into the official task tree before\n  execution. The archive is large; use `limit` or `extra_params.task_ids` for smoke runs.\n- Each selected Claw-Eval task is one EvalScope sample. Official scoring runs once per sample; use EvalScope `repeats`\n  for repeated trials per task and `eval_batch_size` for task-level worker concurrency.\n- Claw-Eval runs with the official Docker sandbox image. If `claw-eval-agent:latest` is missing locally, EvalScope\n  builds it automatically from the cached official `Dockerfile.agent`. The first run can be slow.\n- EvalScope `use_cache` resumes completed task-level samples. Claw-Eval trace JSONL files are stored under\n  `outputs/.../claw_eval/<split>/traces` and converted to EvalScope agent traces for dashboard visualization.\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `claw_eval` |\n| **Dataset ID** | [claw-eval/Claw-Eval](https://modelscope.cn/datasets/claw-eval/Claw-Eval/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `MultiModal`, `MultiTurn` |\n| **Metrics** | `avg_score`, `pass_at_k`, `pass_hat_k`, `error_rate` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 300 |\n| Prompt Length (Mean) | 47.36 chars |\n| Prompt Length (Min/Max) | 30 / 60 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `general` | 161 | 46.47 | 34 | 58 |\n| `multimodal` | 101 | 49.75 | 30 | 60 |\n| `multi_turn` | 38 | 44.79 | 35 | 51 |\n\n## Sample Example\n\n**Subset**: `general`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"a34b7f6b\",\n      \"content\": \"Run Claw-Eval task T001zh_email_triage.\"\n    }\n  ],\n  \"target\": \"\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"subset_key\": \"general\",\n  \"metadata\": {\n    \"task_id\": \"T001zh_email_triage\",\n    \"split\": \"general\",\n    \"task_name\": \"\",\n    \"difficulty\": \"\",\n    \"dataset_id\": \"claw-eval/Claw-Eval\",\n    \"dataset_hub\": \"modelscope\"\n  }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | Optional exact Claw-Eval task ids to run after split filtering. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets claw_eval \\\n    --limit 10  # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['claw_eval'],\n    dataset_args={\n        'claw_eval': {\n            # subset_list: ['general', 'multimodal', 'multi_turn']  # optional, evaluate specific subsets\n            # extra_params: {}  # uses default extra parameters\n        }\n    },\n    limit=10,  # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
+    "zh": "# Claw-Eval\n\n\n## 概述\n\nClaw-Eval 用于评估助手代理在真实个人助理工作流中的表现，这些工作流需要使用工具、访问文件和固定资源（fixtures）、处理多模态输入，并支持模拟用户交互。EvalScope 运行官方指定版本的 Claw-Eval Python 执行器、Docker 沙箱和评分器，同时将每个 Claw-Eval 任务作为标准的 EvalScope 样本进行处理，以支持缓存、重复执行、并行运行、报告生成以及仪表盘轨迹审查。\n\n## 任务描述\n\n- **任务类型**：涉及工具调用、沙箱文件、多模态固定资源及可选模拟用户轮次的代理型个人助理任务。\n- **数据集**：ModelScope 上的 `claw-eval/Claw-Eval`。\n- **子集**：`general`、`multimodal` 和 `multi_turn`；当前 ModelScope 清单包含 300 个任务（161 个 general、101 个 multimodal、38 个 multi_turn）。可通过 `subset_list` 选择子集。\n- **输出**：官方 Claw-Eval 评分与 JSONL 轨迹、EvalScope 样本级评审结果、分组汇总指标，以及仪表盘渲染的代理轨迹。\n\n## 评估说明\n\n- 需要 Python 3.11+，并从指定源代码提交安装官方包：\n  `pip install \"claw-eval[sandbox,mock,web] @ git+https://github.com/claw-eval/claw-eval.git@d3f02d4938ab0832377d90535013def2b1a2fdc0\"`。\n- 安装的包提供 Claw-Eval 执行器 API。EvalScope 同时缓存相同版本的源代码归档，因为 `tasks/` 和 `Dockerfile.agent` 是运行时资产，随后从 ModelScope 加载任务清单和固定资源。\n- 完整的固定资源从 ModelScope 下载（`data/fixtures.tar.gz`），并在执行前链接到官方任务目录中。该归档较大；建议在试运行时使用 `limit` 或 `extra_params.task_ids` 参数。\n- 每个选定的 Claw-Eval 任务对应一个 EvalScope 样本。官方评分对每个样本仅运行一次；如需对同一任务进行多次试验，请使用 EvalScope 的 `repeats` 参数；如需任务级并发执行，请使用 `eval_batch_size`。\n- Claw-Eval 使用官方 Docker 沙箱镜像运行。若本地缺少 `claw-eval-agent:latest` 镜像，EvalScope 会自动基于缓存的官方 `Dockerfile.agent` 构建。首次运行可能较慢。\n- EvalScope 的 `use_cache` 功能可恢复已完成的任务级样本。Claw-Eval 轨迹 JSONL 文件存储在 `outputs/.../claw_eval/<split>/traces` 目录下，并转换为 EvalScope 代理轨迹以供仪表盘可视化。\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `claw_eval` |\n| **数据集ID** | [claw-eval/Claw-Eval](https://modelscope.cn/datasets/claw-eval/Claw-Eval/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `MultiModal`, `MultiTurn` |\n| **指标** | `avg_score`, `pass_at_k`, `pass_hat_k`, `error_rate` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 300 |\n| 提示词长度（平均） | 47.36 字符 |\n| 提示词长度（最小/最大） | 30 / 60 字符 |\n\n**各子集统计数据：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `general` | 161 | 46.47 | 34 | 58 |\n| `multimodal` | 101 | 49.75 | 30 | 60 |\n| `multi_turn` | 38 | 44.79 | 35 | 51 |\n\n## 样例示例\n\n**子集**: `general`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"a34b7f6b\",\n      \"content\": \"Run Claw-Eval task T001zh_email_triage.\"\n    }\n  ],\n  \"target\": \"\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"subset_key\": \"general\",\n  \"metadata\": {\n    \"task_id\": \"T001zh_email_triage\",\n    \"split\": \"general\",\n    \"task_name\": \"\",\n    \"difficulty\": \"\",\n    \"dataset_id\": \"claw-eval/Claw-Eval\",\n    \"dataset_hub\": \"modelscope\"\n  }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | 可选，在子集过滤后精确指定要运行的 Claw-Eval 任务 ID 列表。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets claw_eval \\\n    --limit 10  # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['claw_eval'],\n    dataset_args={\n        'claw_eval': {\n            # subset_list: ['general', 'multimodal', 'multi_turn']  # 可选，评估特定子集\n            # extra_params: {}  # 使用默认额外参数\n        }\n    },\n    limit=10,  # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
+    "content_hash": "96fad09d5529737f9cd0025e2494c9cd",
+    "needs_translation": false
+  },
+  "updated_at": "2026-07-14T17:44:31.169240",
+  "translation_updated_at": "2026-07-14T17:44:38"
+}

evalscope/evalscope/benchmarks/_meta/cmath.json

@@ -130,4 +130,4 @@
   },
   "updated_at": "2026-07-03T16:31:10.424720",
   "translation_updated_at": "2026-07-03T16:31:16"
-}
+}

evalscope/evalscope/benchmarks/_meta/cmmlu.json

@@ -741,4 +741,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.203370",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/cmmmu.json

@@ -1399,4 +1399,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.203423",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/cmmu.json

@@ -431,4 +431,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.203937",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/coin_flip.json

@@ -81,4 +81,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.204932",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/common_voice_15.json

@@ -177,4 +177,4 @@
   },
   "updated_at": "2026-06-23T15:37:29.293476+08:00",
   "translation_updated_at": "2026-06-23T15:50:44+08:00"
-}
+}

evalscope/evalscope/benchmarks/_meta/commonsense_qa.json

@@ -79,4 +79,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.207495",
   "translation_updated_at": "2026-01-28T15:56:15Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/competition_math.json

@@ -119,4 +119,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.213604",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/conll2003.json

@@ -103,4 +103,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.340919",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/conllpp.json

@@ -103,4 +103,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.347147",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/copious.json

@@ -909,4 +909,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.350016",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/cross_ner.json

@@ -159,4 +159,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.352927",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/data_collection.json

@@ -45,4 +45,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.213349",
   "translation_updated_at": "2026-03-05T17:50:46Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/deep_swe.json

@@ -0,0 +1,114 @@
+{
+  "meta": {
+    "pretty_name": "DeepSWE",
+    "dataset_id": "evalscope/deep-swe",
+    "paper_url": null,
+    "tags": [
+      "Coding",
+      "Agent",
+      "MultiTurn"
+    ],
+    "metrics": [
+      "acc"
+    ],
+    "few_shot_num": 0,
+    "eval_split": "test",
+    "train_split": "",
+    "subset_list": [
+      "default"
+    ],
+    "description": "\n## Overview\n\nDeepSWE is a coding-agent benchmark for evaluating repository-level software engineering tasks. EvalScope\nintegrates it through Pier and runs each benchmark sample as one Pier Python API job.\n\n## Task Description\n\n- **Task Type**: Agentic software engineering\n- **Input**: DeepSWE task directory containing task metadata and verifier assets\n- **Output**: A repository patch produced by a Pier built-in agent\n- **Scoring**: Binary verifier reward exposed as `acc`\n\n## Evaluation Notes\n\n- Requires **Python>=3.12**, Docker, and `pip install evalscope[deep_swe]`\n- Dataset defaults to ModelScope `evalscope/deep-swe`\n- DeepSWE runs through Pier's Docker environment in EvalScope\n- Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API\n",
+    "prompt_template": "{question}",
+    "system_prompt": "",
+    "few_shot_prompt_template": "",
+    "aggregation": "mean",
+    "extra_params": {
+      "task_ids": {
+        "type": "list",
+        "description": "Optional list of DeepSWE task ids to evaluate.",
+        "value": []
+      },
+      "languages": {
+        "type": "list",
+        "description": "Optional task language filter from manifest metadata.",
+        "value": []
+      },
+      "categories": {
+        "type": "list",
+        "description": "Optional task category filter from manifest metadata.",
+        "value": []
+      },
+      "sample_seed": {
+        "type": "int",
+        "description": "Optional deterministic shuffle seed applied before limit.",
+        "value": ""
+      },
+      "pier_agent_kwargs": {
+        "type": "dict",
+        "description": "Extra kwargs passed to Pier AgentConfig.kwargs.",
+        "value": {}
+      }
+    },
+    "sandbox_config": {},
+    "category": "agent"
+  },
+  "statistics": {
+    "total_samples": 113,
+    "subset_stats": [
+      {
+        "name": "test",
+        "sample_count": 113,
+        "prompt_length_mean": 2158.07,
+        "prompt_length_min": 471,
+        "prompt_length_max": 5385,
+        "prompt_length_std": 1003.9,
+        "target_length_mean": null
+      }
+    ],
+    "prompt_length": {
+      "mean": 2158.07,
+      "min": 471,
+      "max": 5385,
+      "std": 1003.9
+    },
+    "target_length_mean": null,
+    "computed_at": "2026-07-06T17:32:56.616065"
+  },
+  "sample_example": {
+    "data": {
+      "input": [
+        {
+          "id": "f61040e0",
+          "content": "Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\n\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\n\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\n\n"
+        }
+      ],
+      "target": "",
+      "id": 0,
+      "group_id": 0,
+      "metadata": {
+        "ext_id": "kh701jywhzgddknqwzsq6npjv98226tq",
+        "task_id": "superjson-error-stack-serialization",
+        "display_title": "Add error stack serialization to SuperJSON",
+        "display_description": "Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.",
+        "repo": "flightcontrolhq/superjson",
+        "repository_url": "https://github.com/flightcontrolhq/superjson.git",
+        "original_title": "Error Stack Serialization Support",
+        "category": "feature_request",
+        "language": "typescript",
+        "task_path": "~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization",
+        "task_toml_path": "~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml",
+        "instruction": "Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\n\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\n\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\n\n"
+      }
+    },
+    "subset": "test",
+    "truncated": false
+  },
+  "readme": {
+    "en": "# DeepSWE\n\n\n## Overview\n\nDeepSWE is a coding-agent benchmark for evaluating repository-level software engineering tasks. EvalScope\nintegrates it through Pier and runs each benchmark sample as one Pier Python API job.\n\n## Task Description\n\n- **Task Type**: Agentic software engineering\n- **Input**: DeepSWE task directory containing task metadata and verifier assets\n- **Output**: A repository patch produced by a Pier built-in agent\n- **Scoring**: Binary verifier reward exposed as `acc`\n\n## Evaluation Notes\n\n- Requires **Python>=3.12**, Docker, and `pip install evalscope[deep_swe]`\n- Dataset defaults to ModelScope `evalscope/deep-swe`\n- DeepSWE runs through Pier's Docker environment in EvalScope\n- Use `pier_agent_kwargs={'model_class': 'litellm'}` for OpenAI-compatible providers that do not support Responses API\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `deep_swe` |\n| **Dataset ID** | [evalscope/deep-swe](https://modelscope.cn/datasets/evalscope/deep-swe/summary) |\n| **Paper** | N/A |\n| **Tags** | `Agent`, `Coding`, `MultiTurn` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 113 |\n| Prompt Length (Mean) | 2158.07 chars |\n| Prompt Length (Min/Max) | 471 / 5385 chars |\n\n## Sample Example\n\n**Subset**: `test`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"f61040e0\",\n      \"content\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n    }\n  ],\n  \"target\": \"\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"metadata\": {\n    \"ext_id\": \"kh701jywhzgddknqwzsq6npjv98226tq\",\n    \"task_id\": \"superjson-error-stack-serialization\",\n    \"display_title\": \"Add error stack serialization to SuperJSON\",\n    \"display_description\": \"Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.\",\n    \"repo\": \"flightcontrolhq/superjson\",\n    \"repository_url\": \"https://github.com/flightcontrolhq/superjson.git\",\n    \"original_title\": \"Error Stack Serialization Support\",\n    \"category\": \"feature_request\",\n    \"language\": \"typescript\",\n    \"task_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization\",\n    \"task_toml_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml\",\n    \"instruction\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n  }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\n{question}\n```\n\n## Extra Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | Optional list of DeepSWE task ids to evaluate. |\n| `languages` | `list` | `[]` | Optional task language filter from manifest metadata. |\n| `categories` | `list` | `[]` | Optional task category filter from manifest metadata. |\n| `sample_seed` | `int` | `` | Optional deterministic shuffle seed applied before limit. |\n| `pier_agent_kwargs` | `dict` | `{}` | Extra kwargs passed to Pier AgentConfig.kwargs. |\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets deep_swe \\\n    --limit 10  # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['deep_swe'],\n    dataset_args={\n        'deep_swe': {\n            # extra_params: {}  # uses default extra parameters\n        }\n    },\n    limit=10,  # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
+    "zh": "# DeepSWE\n\n\n## 概述\n\nDeepSWE 是一个用于评估仓库级软件工程任务的编码智能体基准测试。EvalScope 通过 Pier 集成该基准，并将每个基准样本作为一项 Pier Python API 任务运行。\n\n## 任务描述\n\n- **任务类型**：智能体软件工程\n- **输入**：包含任务元数据和验证器资源的 DeepSWE 任务目录\n- **输出**：由 Pier 内置智能体生成的代码仓库补丁\n- **评分方式**：二值验证器奖励，以 `acc` 形式暴露\n\n## 评估说明\n\n- 要求 **Python>=3.12**、Docker，以及执行 `pip install evalscope[deep_swe]`\n- 数据集默认使用 ModelScope 上的 `evalscope/deep-swe`\n- DeepSWE 在 EvalScope 中通过 Pier 的 Docker 环境运行\n- 对于不支持 Responses API 的 OpenAI 兼容提供商，请使用 `pier_agent_kwargs={'model_class': 'litellm'}`\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `deep_swe` |\n| **数据集ID** | [evalscope/deep-swe](https://modelscope.cn/datasets/evalscope/deep-swe/summary) |\n| **论文** | 无 |\n| **标签** | `Agent`, `Coding`, `MultiTurn` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评估划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 113 |\n| 提示词长度（平均） | 2158.07 字符 |\n| 提示词长度（最小/最大） | 471 / 5385 字符 |\n\n## 样例示例\n\n**子集**: `test`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"f61040e0\",\n      \"content\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n    }\n  ],\n  \"target\": \"\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"metadata\": {\n    \"ext_id\": \"kh701jywhzgddknqwzsq6npjv98226tq\",\n    \"task_id\": \"superjson-error-stack-serialization\",\n    \"display_title\": \"Add error stack serialization to SuperJSON\",\n    \"display_description\": \"Add configurable serialization and restoration of error stacks, stack frames, causes, and sanitization in SuperJSON.\",\n    \"repo\": \"flightcontrolhq/superjson\",\n    \"repository_url\": \"https://github.com/flightcontrolhq/superjson.git\",\n    \"original_title\": \"Error Stack Serialization Support\",\n    \"category\": \"feature_request\",\n    \"language\": \"typescript\",\n    \"task_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization\",\n    \"task_toml_path\": \"~/.cache/evalscope/deep_swe/snapshots/evalscope/deep-swe/tasks/superjson-error-stack-serialization/task.toml\",\n    \"instruction\": \"Add a new `errorStack` constructor option to SuperJSON. Omitting it leaves existing Error behavior unchanged.\\n\\nThe option shape is `{ mode?, normalizeNewlines?, trimLeadingWhitespace?, maxStackLines?, stripInternalFrames?, redactPaths?, inclu ... [TRUNCATED 3577 chars] ... ): Processor | undefined`. `normalizeErrorStackOptions` returns `undefined` for any non-object input (`null`, `undefined`, strings).\\n\\nBefore writing, read through the existing error serialization logic and the `allowedErrorProps` mechanism.\\n\\n\"\n  }\n}\n```\n\n## 提示模板\n\n**提示模板:**\n```text\n{question}\n```\n\n## 额外参数\n\n| 参数 | 类型 | 默认值 | 描述 |\n|-----------|------|---------|-------------|\n| `task_ids` | `list` | `[]` | 可选的 DeepSWE 任务 ID 列表，用于指定评估范围。 |\n| `languages` | `list` | `[]` | 可选的任务语言过滤器，基于清单元数据。 |\n| `categories` | `list` | `[]` | 可选的任务类别过滤器，基于清单元数据。 |\n| `sample_seed` | `int` | `` | 可选的确定性打乱种子，在限制样本数量前应用。 |\n| `pier_agent_kwargs` | `dict` | `{}` | 传递给 Pier AgentConfig.kwargs 的额外关键字参数。 |\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets deep_swe \\\n    --limit 10  # 正式评估时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['deep_swe'],\n    dataset_args={\n        'deep_swe': {\n            # extra_params: {}  # 使用默认额外参数\n        }\n    },\n    limit=10,  # 正式评估时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
+    "content_hash": "54626e00b5f66f41f2353bba4826cd3c",
+    "needs_translation": false
+  },
+  "updated_at": "2026-07-06T17:32:58.309507",
+  "translation_updated_at": "2026-07-06T17:33:00"
+}

evalscope/evalscope/benchmarks/_meta/docmath.json

@@ -105,4 +105,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.213621",
   "translation_updated_at": "2026-01-28T17:21:59Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/docvqa.json

@@ -151,4 +151,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.214406",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/drivel_binary.json

@@ -80,4 +80,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.222850",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/drivel_multilabel.json

@@ -91,4 +91,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.223704",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/drivel_selection.json

@@ -87,4 +87,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.225038",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/drivel_writing.json

@@ -86,4 +86,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.225015",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/drop.json

@@ -120,4 +120,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.227072",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/emb_spatial_bench.json

@@ -0,0 +1,312 @@
+{
+  "meta": {
+    "pretty_name": "EmbSpatial-Bench",
+    "dataset_id": "evalscope/EmbSpatial-Bench",
+    "paper_url": "https://aclanthology.org/2024.acl-short.33/",
+    "tags": [
+      "MultiModal",
+      "Reasoning",
+      "MCQ"
+    ],
+    "metrics": [
+      "acc"
+    ],
+    "few_shot_num": 0,
+    "eval_split": "test",
+    "train_split": "",
+    "subset_list": [
+      "close",
+      "far",
+      "above",
+      "under",
+      "left",
+      "right"
+    ],
+    "description": "\n## Overview\n\nEmbSpatial-Bench is a benchmark for evaluating embodied spatial understanding of large vision-language models (LVLMs). The benchmark is automatically derived from embodied scenes and covers 6 spatial relationships from an egocentric perspective: **close**, **far**, **above**, **under**, **left**, and **right**.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Visual Question Answering (VQA)\n- **Input**: An egocentric RGB image + a spatial reasoning question with 4 candidate answers\n- **Output**: A single letter (A / B / C / D) identifying the correct object or spatial relationship\n- **Domains**: Embodied AI, spatial reasoning (MP3D and AI2Thor environments)\n\n## Key Features\n\n- 3,640 human-verified evaluation questions derived from two embodied environments (MP3D and AI2Thor)\n- 6 spatial relation categories: close, far, above, under, left, right\n- Each question requires selecting the most spatially accurate answer from 4 options\n- Designed to expose the gap between current LVLMs and qualified embodied intelligence\n\n## Evaluation Notes\n\n- Default evaluation uses the **embspatial_bench.json** file (3,640 samples)\n- Primary metric: **Accuracy** (acc)\n- Answer indices are 0-based in the dataset (0 → A, 1 → B, 2 → C, 3 → D)\n- Images are stored as JPEG base64 strings in the JSON file\n- Subsets are organized by the `relation` field (6 spatial categories)\n- [Paper](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n",
+    "prompt_template": "Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}",
+    "system_prompt": "",
+    "few_shot_prompt_template": "",
+    "aggregation": "mean",
+    "extra_params": {},
+    "sandbox_config": {},
+    "category": "vlm"
+  },
+  "statistics": {
+    "total_samples": 3640,
+    "subset_stats": [
+      {
+        "name": "close",
+        "sample_count": 612,
+        "prompt_length_mean": 293.54,
+        "prompt_length_min": 274,
+        "prompt_length_max": 326,
+        "prompt_length_std": 10.6,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 612,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      },
+      {
+        "name": "far",
+        "sample_count": 594,
+        "prompt_length_mean": 292.68,
+        "prompt_length_min": 265,
+        "prompt_length_max": 326,
+        "prompt_length_std": 12.55,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 594,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      },
+      {
+        "name": "above",
+        "sample_count": 596,
+        "prompt_length_mean": 405.61,
+        "prompt_length_min": 360,
+        "prompt_length_max": 486,
+        "prompt_length_std": 23.28,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 596,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      },
+      {
+        "name": "under",
+        "sample_count": 602,
+        "prompt_length_mean": 404.35,
+        "prompt_length_min": 350,
+        "prompt_length_max": 490,
+        "prompt_length_std": 22.27,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 602,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      },
+      {
+        "name": "left",
+        "sample_count": 616,
+        "prompt_length_mean": 408.86,
+        "prompt_length_min": 359,
+        "prompt_length_max": 486,
+        "prompt_length_std": 21.53,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 616,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      },
+      {
+        "name": "right",
+        "sample_count": 620,
+        "prompt_length_mean": 409.47,
+        "prompt_length_min": 352,
+        "prompt_length_max": 481,
+        "prompt_length_std": 21.76,
+        "target_length_mean": 1,
+        "multimodal": {
+          "has_images": true,
+          "has_audio": false,
+          "has_video": false,
+          "image": {
+            "count_total": 620,
+            "count_per_sample": {
+              "min": 1,
+              "max": 1,
+              "mean": 1
+            },
+            "resolutions": [
+              "1296x968",
+              "300x300",
+              "640x480"
+            ],
+            "resolution_range": {
+              "min": "300x300",
+              "max": "1296x968"
+            },
+            "formats": [
+              "jpeg"
+            ]
+          }
+        }
+      }
+    ],
+    "prompt_length": {
+      "mean": 369.34,
+      "min": 265,
+      "max": 490,
+      "std": 57.07
+    },
+    "target_length_mean": 1,
+    "computed_at": "2026-07-06T17:23:58.082817",
+    "multimodal": {
+      "has_images": true,
+      "has_audio": false,
+      "has_video": false,
+      "image": {
+        "count_total": 3640,
+        "count_per_sample": {
+          "min": 1,
+          "max": 1,
+          "mean": 1
+        },
+        "resolutions": [
+          "1296x968",
+          "300x300",
+          "640x480"
+        ],
+        "resolution_range": {
+          "min": "300x300",
+          "max": "1296x968"
+        },
+        "formats": [
+          "jpeg"
+        ]
+      }
+    }
+  },
+  "sample_example": {
+    "data": {
+      "input": [
+        {
+          "id": "3f406c29",
+          "content": [
+            {
+              "image": "[BASE64_IMAGE: jpeg, ~35.2KB]"
+            },
+            {
+              "text": "Among the listed objects, which one is closest to your current location in the image?\n(A) table\n(B) towel\n(C) door\n(D) basket\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D."
+            }
+          ]
+        }
+      ],
+      "target": "D",
+      "id": 0,
+      "group_id": 0,
+      "subset_key": "close",
+      "metadata": {
+        "question_id": "mp3d_0",
+        "relation": "close",
+        "data_source": "mp3d"
+      }
+    },
+    "subset": "close",
+    "truncated": false
+  },
+  "readme": {
+    "en": "# EmbSpatial-Bench\n\n\n## Overview\n\nEmbSpatial-Bench is a benchmark for evaluating embodied spatial understanding of large vision-language models (LVLMs). The benchmark is automatically derived from embodied scenes and covers 6 spatial relationships from an egocentric perspective: **close**, **far**, **above**, **under**, **left**, and **right**.\n\n## Task Description\n\n- **Task Type**: Multiple-Choice Visual Question Answering (VQA)\n- **Input**: An egocentric RGB image + a spatial reasoning question with 4 candidate answers\n- **Output**: A single letter (A / B / C / D) identifying the correct object or spatial relationship\n- **Domains**: Embodied AI, spatial reasoning (MP3D and AI2Thor environments)\n\n## Key Features\n\n- 3,640 human-verified evaluation questions derived from two embodied environments (MP3D and AI2Thor)\n- 6 spatial relation categories: close, far, above, under, left, right\n- Each question requires selecting the most spatially accurate answer from 4 options\n- Designed to expose the gap between current LVLMs and qualified embodied intelligence\n\n## Evaluation Notes\n\n- Default evaluation uses the **embspatial_bench.json** file (3,640 samples)\n- Primary metric: **Accuracy** (acc)\n- Answer indices are 0-based in the dataset (0 → A, 1 → B, 2 → C, 3 → D)\n- Images are stored as JPEG base64 strings in the JSON file\n- Subsets are organized by the `relation` field (6 spatial categories)\n- [Paper](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n\n\n## Properties\n\n| Property | Value |\n|----------|-------|\n| **Benchmark Name** | `emb_spatial_bench` |\n| **Dataset ID** | [evalscope/EmbSpatial-Bench](https://modelscope.cn/datasets/evalscope/EmbSpatial-Bench/summary) |\n| **Paper** | [Paper](https://aclanthology.org/2024.acl-short.33/) |\n| **Tags** | `MCQ`, `MultiModal`, `Reasoning` |\n| **Metrics** | `acc` |\n| **Default Shots** | 0-shot |\n| **Evaluation Split** | `test` |\n\n\n## Data Statistics\n\n| Metric | Value |\n|--------|-------|\n| Total Samples | 3,640 |\n| Prompt Length (Mean) | 369.34 chars |\n| Prompt Length (Min/Max) | 265 / 490 chars |\n\n**Per-Subset Statistics:**\n\n| Subset | Samples | Prompt Mean | Prompt Min | Prompt Max |\n|--------|---------|-------------|------------|------------|\n| `close` | 612 | 293.54 | 274 | 326 |\n| `far` | 594 | 292.68 | 265 | 326 |\n| `above` | 596 | 405.61 | 360 | 486 |\n| `under` | 602 | 404.35 | 350 | 490 |\n| `left` | 616 | 408.86 | 359 | 486 |\n| `right` | 620 | 409.47 | 352 | 481 |\n\n**Image Statistics:**\n\n| Metric | Value |\n|--------|-------|\n| Total Images | 3,640 |\n| Images per Sample | min: 1, max: 1, mean: 1 |\n| Resolution Range | 300x300 - 1296x968 |\n| Formats | jpeg |\n\n\n## Sample Example\n\n**Subset**: `close`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"3f406c29\",\n      \"content\": [\n        {\n          \"image\": \"[BASE64_IMAGE: jpeg, ~35.2KB]\"\n        },\n        {\n          \"text\": \"Among the listed objects, which one is closest to your current location in the image?\\n(A) table\\n(B) towel\\n(C) door\\n(D) basket\\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D.\"\n        }\n      ]\n    }\n  ],\n  \"target\": \"D\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"subset_key\": \"close\",\n  \"metadata\": {\n    \"question_id\": \"mp3d_0\",\n    \"relation\": \"close\",\n    \"data_source\": \"mp3d\"\n  }\n}\n```\n\n## Prompt Template\n\n**Prompt Template:**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## Usage\n\n### Using CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets emb_spatial_bench \\\n    --limit 10  # Remove this line for formal evaluation\n```\n\n### Using Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['emb_spatial_bench'],\n    dataset_args={\n        'emb_spatial_bench': {\n            # subset_list: ['close', 'far', 'above']  # optional, evaluate specific subsets\n        }\n    },\n    limit=10,  # Remove this line for formal evaluation\n)\n\nrun_task(task_cfg=task_cfg)\n```\n\n\n",
+    "zh": "# EmbSpatial-Bench\n\n\n## 概述\n\nEmbSpatial-Bench 是一个用于评估大视觉语言模型（LVLMs）具身空间理解能力的基准测试。该基准测试从具身场景中自动构建，涵盖从第一人称视角出发的 6 种空间关系：**close**（近）、**far**（远）、**above**（上）、**under**（下）、**left**（左）和 **right**（右）。\n\n## 任务描述\n\n- **任务类型**：多项选择视觉问答（VQA）\n- **输入**：一张第一人称 RGB 图像 + 一个包含 4 个候选答案的空间推理问题\n- **输出**：单个字母（A / B / C / D），标识正确的物体或空间关系\n- **领域**：具身人工智能、空间推理（MP3D 和 AI2Thor 环境）\n\n## 主要特点\n\n- 包含 3,640 个人工验证的评测问题，源自两个具身环境（MP3D 和 AI2Thor）\n- 涵盖 6 类空间关系：close、far、above、under、left、right\n- 每个问题需从 4 个选项中选出空间上最准确的答案\n- 旨在揭示当前 LVLM 与合格具身智能之间的差距\n\n## 评测说明\n\n- 默认评测使用 **embspatial_bench.json** 文件（共 3,640 个样本）\n- 主要指标：**准确率**（acc）\n- 数据集中答案索引为 0 起始（0 → A，1 → B，2 → C，3 → D）\n- 图像以 JPEG base64 字符串形式存储在 JSON 文件中\n- 子集按 `relation` 字段组织（6 种空间关系类别）\n- [论文](https://aclanthology.org/2024.acl-short.33/) | [GitHub](https://github.com/mengfeidu/EmbSpatial-Bench)\n\n\n## 属性\n\n| 属性 | 值 |\n|----------|-------|\n| **基准测试名称** | `emb_spatial_bench` |\n| **数据集ID** | [evalscope/EmbSpatial-Bench](https://modelscope.cn/datasets/evalscope/EmbSpatial-Bench/summary) |\n| **论文** | [Paper](https://aclanthology.org/2024.acl-short.33/) |\n| **标签** | `MCQ`, `MultiModal`, `Reasoning` |\n| **指标** | `acc` |\n| **默认示例数** | 0-shot |\n| **评测划分** | `test` |\n\n\n## 数据统计\n\n| 指标 | 值 |\n|--------|-------|\n| 总样本数 | 3,640 |\n| 提示词长度（平均） | 369.34 字符 |\n| 提示词长度（最小/最大） | 265 / 490 字符 |\n\n**各子集统计信息：**\n\n| 子集 | 样本数 | 提示词平均长度 | 提示词最小长度 | 提示词最大长度 |\n|--------|---------|-------------|------------|------------|\n| `close` | 612 | 293.54 | 274 | 326 |\n| `far` | 594 | 292.68 | 265 | 326 |\n| `above` | 596 | 405.61 | 360 | 486 |\n| `under` | 602 | 404.35 | 350 | 490 |\n| `left` | 616 | 408.86 | 359 | 486 |\n| `right` | 620 | 409.47 | 352 | 481 |\n\n**图像统计信息：**\n\n| 指标 | 值 |\n|--------|-------|\n| 图像总数 | 3,640 |\n| 每样本图像数 | 最小: 1, 最大: 1, 平均: 1 |\n| 分辨率范围 | 300x300 - 1296x968 |\n| 格式 | jpeg |\n\n\n## 样例示例\n\n**子集**: `close`\n\n```json\n{\n  \"input\": [\n    {\n      \"id\": \"3f406c29\",\n      \"content\": [\n        {\n          \"image\": \"[BASE64_IMAGE: jpeg, ~35.2KB]\"\n        },\n        {\n          \"text\": \"Among the listed objects, which one is closest to your current location in the image?\\n(A) table\\n(B) towel\\n(C) door\\n(D) basket\\nAnswer with only the letter of the correct option. The last line of your response should be of the format: ANSWER: [LETTER] where LETTER is one of A, B, C, D.\"\n        }\n      ]\n    }\n  ],\n  \"target\": \"D\",\n  \"id\": 0,\n  \"group_id\": 0,\n  \"subset_key\": \"close\",\n  \"metadata\": {\n    \"question_id\": \"mp3d_0\",\n    \"relation\": \"close\",\n    \"data_source\": \"mp3d\"\n  }\n}\n```\n\n## 提示模板\n\n**提示模板：**\n```text\nAnswer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of {letters}. Think step by step before answering.\n\n{question}\n\n{choices}\n```\n\n## 使用方法\n\n### 使用 CLI\n\n```bash\nevalscope eval \\\n    --model YOUR_MODEL \\\n    --api-url OPENAI_API_COMPAT_URL \\\n    --api-key EMPTY_TOKEN \\\n    --datasets emb_spatial_bench \\\n    --limit 10  # 正式评测时请删除此行\n```\n\n### 使用 Python\n\n```python\nfrom evalscope import run_task\nfrom evalscope.config import TaskConfig\n\ntask_cfg = TaskConfig(\n    model='YOUR_MODEL',\n    api_url='OPENAI_API_COMPAT_URL',\n    api_key='EMPTY_TOKEN',\n    datasets=['emb_spatial_bench'],\n    dataset_args={\n        'emb_spatial_bench': {\n            # subset_list: ['close', 'far', 'above']  # 可选，用于评测特定子集\n        }\n    },\n    limit=10,  # 正式评测时请删除此行\n)\n\nrun_task(task_cfg=task_cfg)\n```",
+    "content_hash": "1a472c56595a22ddb88cd289b3a34111",
+    "needs_translation": false
+  },
+  "updated_at": "2026-07-06T17:58:15.970262",
+  "translation_updated_at": "2026-07-06T17:58:37"
+}

evalscope/evalscope/benchmarks/_meta/eq_bench.json

@@ -91,4 +91,4 @@
   },
   "updated_at": "2026-01-28T17:31:32.228078",
   "translation_updated_at": "2026-01-28T16:09:53Z"
-}
+}

evalscope/evalscope/benchmarks/_meta/erqa.json

@@ -442,4 +442,4 @@
   },
   "updated_at": "2026-07-02T19:24:50.476503",
   "translation_updated_at": "2026-07-02T19:26:43"
-}
+}